diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index d94f837d59..a2f8a0524a 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -158,7 +158,7 @@ steps: # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml # * test/tag_for_hosted.sh - name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0'] + args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0'] id: 'pullMetadataServer' - name: 'gcr.io/cloud-builders/docker' args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance'] diff --git a/.github/stale.yml b/.github/stale.yml index ad4197e7f0..7232a69fed 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -18,4 +18,4 @@ markComment: > # Comment to post when closing a stale issue. Set to `false` to disable closeComment: > This issue has been automatically closed because it has not had recent - activity. Please comment "/reopen" to reopen it. + activity. Please comment "/reopen" to reopen it. \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..1d4fa1c740 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,39 @@ +# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. +# +# You can adjust the behavior by modifying this file. +# For more information, see: +# https://github.com/actions/stale +name: Mark stale issues and pull requests + +on: + schedule: + - cron: '39 7 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v5 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: -1 + days-before-close: -1 + days-before-issue-stale: 90 + days-before-issue-close: 90 + stale-issue-message: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + close-issue-message: > + This issue has been automatically closed because it has not had recent + activity. Please comment "/reopen" to reopen it. + stale-issue-label: 'lifecycle/stale' + exempt-issue-labels: lifecycle/frozen + exempt-pr-labels: lifecycle/frozen + # Learn more about operations: https://github.com/actions/stale#operations-per-run. + operations-per-run: 200 \ No newline at end of file diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml index 0c0eee6707..591c3ed198 100644 --- a/.release.cloudbuild.yaml +++ b/.release.cloudbuild.yaml @@ -259,14 +259,14 @@ steps: # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml # * test/tag_for_hosted.sh - name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0'] + args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0'] id: 'pullMetadataServer' - name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0', 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME'] + args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0', 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME'] id: 'tagMetadataServerForMarketplace' waitFor: ['pullMetadataServer'] - name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0', 'gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$TAG_NAME'] + args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0', 'gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$TAG_NAME'] id: 'tagMetadataServerForMarketplaceTest' waitFor: ['pullMetadataServer'] - id: 'tagMetadataServerForMarketplaceMajorMinor' @@ -276,8 +276,8 @@ steps: args: - -ceux - | - docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0 gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver) - docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0 gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver) + docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver) + docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver) docker push gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver) docker push gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c4a74290a..c65dcbe724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,160 @@ # Changelog +### [2.0.3](https://github.com/kubeflow/pipelines/compare/2.0.2...2.0.3) (2023-10-27) + + +### Features + +* **backend:** Support consuming parent DAG input artifact ([\#10162](https://github.com/kubeflow/pipelines/issues/10162)) ([52f5cf5](https://github.com/kubeflow/pipelines/commit/52f5cf51c4a6c233aae57125561c0fc95c4fd20f)) +* **backend:** Update driver and launcher images ([\#10164](https://github.com/kubeflow/pipelines/issues/10164)) ([c0093ec](https://github.com/kubeflow/pipelines/commit/c0093ecef6bc5f056efa135d019267327115d79d)) +* **components:** [endpoint_batch_predict] Initialize component ([0d75611](https://github.com/kubeflow/pipelines/commit/0d7561199751e83b4d7e1603c3d32d4088a7e208)) +* **components:** [text2sql] Generate column names by model batch predict ([1bee8be](https://github.com/kubeflow/pipelines/commit/1bee8be071a91f44c0129837c381863327cb337d)) +* **components:** [text2sql] Generate table names by model batch prediction ([ebb4245](https://github.com/kubeflow/pipelines/commit/ebb42450d0b07eaa8de35a3f6b70eacb5f26f0d8)) +* **components:** [text2sql] Implement preprocess component logic ([21079b5](https://github.com/kubeflow/pipelines/commit/21079b5910e597a38b67853f3ecfb3929344371e)) +* **components:** [text2sql] Initialize preprocess component and integrate with text2sql pipeline ([9aa750e](https://github.com/kubeflow/pipelines/commit/9aa750e62f6e225d037ecdda9bf7cab95f05675d)) +* **components:** [text2sql] Initialize evaluation component ([ea93979](https://github.com/kubeflow/pipelines/commit/ea93979eed02e131bd20180da149b9465670dfe1)) +* **components:** [text2sql] Initialize validate and process component ([633ddeb](https://github.com/kubeflow/pipelines/commit/633ddeb07e9212d2e373dba8d20a0f6d67ab037d)) +* **components:** Add ability to preprocess chat llama datasets to `_implementation.llm.chat_dataset_preprocessor` ([99fd201](https://github.com/kubeflow/pipelines/commit/99fd2017a76660f30d0a04b71542cbef45783633)) +* **components:** Add question_answer support for AutoSxS default instructions ([412216f](https://github.com/kubeflow/pipelines/commit/412216f832a848bfc61ce289aed819d7f2860fdd)) +* **components:** Add sliced evaluation metrics support for custom and unstructured AutoML models in evaluation feature attribution pipeline ([d8a0660](https://github.com/kubeflow/pipelines/commit/d8a0660df525f5695015e507e981bceff836dd3d)) +* **components:** Add sliced evaluation metrics support for custom and unstructured AutoML models in evaluation pipeline ([0487f9a](https://github.com/kubeflow/pipelines/commit/0487f9a8b1d8ab0d96d757bd4b598ffd353ecc81)) +* **components:** add support for customizing model_parameters in LLM eval text generation and LLM eval text classification pipelines ([d53ddda](https://github.com/kubeflow/pipelines/commit/d53dddab1c8a042e58e06ff6eb38be82fefddb0a)) +* **components:** Make `model_checkpoint` optional for `preview.llm.infer_pipeline` ([e8fb699](https://github.com/kubeflow/pipelines/commit/e8fb6990dfdf036c941c522f9b384ff679b38ca6)) +* **components:** migrate `DataflowFlexTemplateJobOp` to GA namespace (now `v1.dataflow.DataflowFlexTemplateJobOp`) ([faba922](https://github.com/kubeflow/pipelines/commit/faba9223ee846d459f7bb497a6faa3c153dcf430)) +* **components:** Set display names for SFT, RLHF and LLM inference pipelines ([1386a82](https://github.com/kubeflow/pipelines/commit/1386a826ba2bcdbc19eb2007ca43f6acd1031e4d)) +* **components:** Support service account in kubeflow model_batch_predict component ([1682ce8](https://github.com/kubeflow/pipelines/commit/1682ce8adeb2c55a155588eae7492b2f0a8b783a)) +* **components:** Update image tag used by llm pipelines ([4d71fda](https://github.com/kubeflow/pipelines/commit/4d71fdac3fc92dd4d54c6be3a28725667b8f3c5e)) +* **sdk:** support a Pythonic artifact authoring style ([\#9932](https://github.com/kubeflow/pipelines/issues/9932)) ([8d00d0e](https://github.com/kubeflow/pipelines/commit/8d00d0eb9a1442ed994b6a90acea88604efc6423)) +* **sdk:** support collecting outputs from conditional branches using `dsl.OneOf` ([\#10067](https://github.com/kubeflow/pipelines/issues/10067)) ([2d3171c](https://github.com/kubeflow/pipelines/commit/2d3171cbfec626055e59b8a58ce83fb54ecad113)) + + +### Bug Fixes + +* **components:** [text2sql] Turn model_inference_results_path to model_inference_results_directory and remove duplicate comment ([570e56d](https://github.com/kubeflow/pipelines/commit/570e56dd09af32e173cf041eed7497e4533ec186)) +* **frontend:** Replace twitter artifactory endpoint with npm endpoint. ([\#10099](https://github.com/kubeflow/pipelines/issues/10099)) ([da6a360](https://github.com/kubeflow/pipelines/commit/da6a3601468282c0592eae8e89a3d97b982e2d43)) +* **sdk:** fix bug when `dsl.importer` argument is provided by loop variable ([\#10116](https://github.com/kubeflow/pipelines/issues/10116)) ([73d51c8](https://github.com/kubeflow/pipelines/commit/73d51c8a23afad97efb6d7e7436c081fa22ce24d)) +* **sdk:** Fix OOB for IPython and refactor. Closes [\#10075](https://github.com/kubeflow/pipelines/issues/10075). ([\#10094](https://github.com/kubeflow/pipelines/issues/10094)) ([c903271](https://github.com/kubeflow/pipelines/commit/c9032716ab2013df56cb1078a703d48ed8e36fb4)) +* **sdk:** type annotation for client credentials ([\#10158](https://github.com/kubeflow/pipelines/issues/10158)) ([02e00e8](https://github.com/kubeflow/pipelines/commit/02e00e8439e9753dbf82856ac9c5a7cec8ce3243)) + + +### Other Pull Requests + +* feat(components) Extend kserve component ([\#10136](https://github.com/kubeflow/pipelines/issues/10136)) ([2054b7c](https://github.com/kubeflow/pipelines/commit/2054b7c45d4831c787115563c8be0048abcb9be1)) +* No public description ([0e240db](https://github.com/kubeflow/pipelines/commit/0e240db39799cb0afbd8c7f982ffdd4f9eb58121)) + +### [2.0.2](https://github.com/kubeflow/pipelines/compare/2.0.0...2.0.2) (2023-10-11) + + +### Features + +* **backend:** add postgres initialization ([\#9798](https://github.com/kubeflow/pipelines/issues/9798)) ([e1f0c01](https://github.com/kubeflow/pipelines/commit/e1f0c010f80031ea09af69f9bbedf2e24509605f)) +* **backend:** Added metrics to be collected from failed/successful workflows ([\#9576](https://github.com/kubeflow/pipelines/issues/9576)) ([5835824](https://github.com/kubeflow/pipelines/commit/5835824e9cca76af70b733e7d494bb4bbdd8e2b7)) +* **backend:** enforce SA Token based auth b/w Persistence Agent and Pipeline API Server ([\#9957](https://github.com/kubeflow/pipelines/issues/9957)) ([760c158](https://github.com/kubeflow/pipelines/commit/760c1589edbe58bbd77611222a66a17b371a0d08)) +* **backend:** Update driver and launcher images ([\#10076](https://github.com/kubeflow/pipelines/issues/10076)) ([3c5f62a](https://github.com/kubeflow/pipelines/commit/3c5f62a6a32669736f73f54c79ab6b0d04349c6f)) +* **chore:** Change AutoML Vision Error Analysis pipeline names ([450e910](https://github.com/kubeflow/pipelines/commit/450e9108172b5a4eb76abb6647bb65661581747a)) +* **components:** [text2sql] Initialize text2sql pipeline ([72e7f96](https://github.com/kubeflow/pipelines/commit/72e7f964c542ffd0bc485da9fce9ca5b047c3ab7)) +* **components:** add `persistent_resource_id` to preview GCPC custom job components/utils ([fc1f12b](https://github.com/kubeflow/pipelines/commit/fc1f12b7bd2f28390c838abcf3dd020723ad573a)) +* **components:** Add AutoML image training job v1 remote runner ([df4bc46](https://github.com/kubeflow/pipelines/commit/df4bc46725798d27a32c3935e48dae2384e7d4b9)) +* **components:** Add Feature Attribution components to _implementation/model_evaluation. Add LLM Eval text generation and text classification pipelines to preview namespace init file ([f454a86](https://github.com/kubeflow/pipelines/commit/f454a86177b85b5cc11a7c57f63fa7f03f45604c)) +* **components:** Add helper functions to create slice_specs and bias_configs ([95901c8](https://github.com/kubeflow/pipelines/commit/95901c88302c61e6cdc33ddd2cd96ab65663e881)) +* **components:** Add LLM implementation component that uploads tensorboard metrics after training ([b273aab](https://github.com/kubeflow/pipelines/commit/b273aabb894338c85093b0fb564bb5b3094e36b7)) +* **components:** Add main entry and command-line flags for Templated Custom Job Launcher ([f8f01bc](https://github.com/kubeflow/pipelines/commit/f8f01bcd08ba30bb8ac902843468984fdc662033)) +* **components:** Add rlhf and infer pipelines to preview/llm. Add llm related components to _implementation/llm ([611298a](https://github.com/kubeflow/pipelines/commit/611298a8ee68f406e09009debb909a44de0ae99e)) +* **components:** Add sampling_strategy parameter to bulk inferrer to support different strategy. By default, we use greedy ([e21174f](https://github.com/kubeflow/pipelines/commit/e21174f94aa75f48b6ae99f4c4b64f82d91bffd9)) +* **components:** Add support for customizing evaluation_display_name in model evaluation pipelines ([e8b8450](https://github.com/kubeflow/pipelines/commit/e8b8450e0a9501eca130b02d2cf2995b994d02c3)) +* **components:** add Vertex RAI safety bias evaluation pipeline ([b630d5c](https://github.com/kubeflow/pipelines/commit/b630d5c8ae7559be0011e67f01e3aec1946ef765)) +* **components:** Add vision data converter component to preview ([8f07661](https://github.com/kubeflow/pipelines/commit/8f07661ae96921a9245c43f2385a92ebcf69978c)) +* **components:** Allow ImportModelEvaluationOp to take LLM metrics from --metrics and --problem_type parameters ([d331ca0](https://github.com/kubeflow/pipelines/commit/d331ca0204359d67e03fcd9b903ed2eff2b299a6)) +* **components:** Components for Embedding Eval pipeline ([562cd29](https://github.com/kubeflow/pipelines/commit/562cd299cc25244b9b3d900cd8c6cb86142f8326)) +* **components:** define new GCPC Model Eval component for LLM Text Generation ([a634eef](https://github.com/kubeflow/pipelines/commit/a634eef3ec541ee64eb0220d5db12b82f682479e)) +* **components:** Embedding eval pipeline for experimental launch ([cac1856](https://github.com/kubeflow/pipelines/commit/cac185653317326459ff1f4a107b86c29aedaf59)) +* **components:** fork a subset of `v1` `custom_job` and `gcp_launcher` container code to `preview` ([abf05f4](https://github.com/kubeflow/pipelines/commit/abf05f48191b214bf5e993cd4cc725ff793d544c)) +* **components:** Implement `create_templated_custom_job` for Templated Custom Job Launcher ([e307545](https://github.com/kubeflow/pipelines/commit/e307545e689516c1249d1211e4131db49a346ffd)) +* **components:** Implement chunking for embedding evaluation pipeline ([0ced6ec](https://github.com/kubeflow/pipelines/commit/0ced6ec7d2846faefc655bad5ea549f81cfcd373)) +* **components:** Implement helper functions for Jinja2-based Templated Custom Job Launcher ([8518e95](https://github.com/kubeflow/pipelines/commit/8518e95efcdb2c78a4ae719af66b72caac082267)) +* **components:** Implement LLM Safety Bias Component & E2E tests ([ae804f4](https://github.com/kubeflow/pipelines/commit/ae804f471cb5ad7e4ba70ee44bf958a4a909d2a9)) +* **components:** Implement the chunking component ([a76e385](https://github.com/kubeflow/pipelines/commit/a76e385b20b30f9a974139bc4d6d04f8517441f2)) +* **components:** Implement timestamp filtering for chunking ([5c44143](https://github.com/kubeflow/pipelines/commit/5c44143742213c11821d4775d7fda50724747032)) +* **components:** internal change ([0670337](https://github.com/kubeflow/pipelines/commit/067033762db315f83b84cfe1d6dc039c96a0e9f2)) +* **components:** Internal change ([ff90cea](https://github.com/kubeflow/pipelines/commit/ff90ceae9a4f403a14dd01c5468068c6079d511f)) +* **components:** Internal change ([1dc8453](https://github.com/kubeflow/pipelines/commit/1dc84534d406e1b3fd683fbc1504587e22d5f5d8)) +* **components:** Internal change ([f80d2b3](https://github.com/kubeflow/pipelines/commit/f80d2b30e4c7d05c0f511f3bf3d6dd102a9a578a)) +* **components:** Internal change ([f8c1f9c](https://github.com/kubeflow/pipelines/commit/f8c1f9cf21d2472017c1a07319d3fab4c22fa7b5)) +* **components:** Metric importing for embedding evaluation ([47f1147](https://github.com/kubeflow/pipelines/commit/47f11475c5b19cc99a49962527723fff5f85aa5a)) +* **components:** Move model & data bias components to preview ([314daa4](https://github.com/kubeflow/pipelines/commit/314daa4a852916c098cd4c126fce8745f4778deb)) +* **components:** Output imported evaluation resource name in ImportModelEvaluationOp ([c27d23a](https://github.com/kubeflow/pipelines/commit/c27d23a79455e18dc6ad362d2ee7353028f32ca6)) +* **components:** Release new model evaluation image versions ([9cf92c3](https://github.com/kubeflow/pipelines/commit/9cf92c31619111ba218a876c292ca4f2a45096c6)) +* **components:** Review and update batch_predict_job GCPC docstrings ([ea7a5ef](https://github.com/kubeflow/pipelines/commit/ea7a5efb97e5aa14dbf58ab55aa2f68b1ddc5941)) +* **components:** Support multiple chunking functions ([4945e2c](https://github.com/kubeflow/pipelines/commit/4945e2cfc5848898727e608ab2f9c607bb99dc92)) +* **components:** Switch v1 AutoMLImageTrainingJob to use the pipeline remote runner ([15c24e3](https://github.com/kubeflow/pipelines/commit/15c24e344f4c624d151116e548c470be935b8e70)) +* **components:** Update container URIs for embedding eval components ([f43272d](https://github.com/kubeflow/pipelines/commit/f43272dee8c40563ee05c07d9e1de56c4ba7c08f)) +* **components:** Update default image tag used by LLM implementation components ([b31d8a5](https://github.com/kubeflow/pipelines/commit/b31d8a57ef5db67a8cd782d7ab60f7e5b131ae7a)) +* **components:** Update embedding pipeline containers to use llm-pipeline image ([1f37243](https://github.com/kubeflow/pipelines/commit/1f37243ec1d1a49af25f7de38b22b068edc18705)) +* **components:** Update policy to reward model name mapping in function based component in _implementation/llm ([110e082](https://github.com/kubeflow/pipelines/commit/110e0824812883b74c73b26603a78d8cc00548d5)) +* **components:** Update RAI safety component with latest image ([af753dc](https://github.com/kubeflow/pipelines/commit/af753dc645ea2630a07dce3c0a1287ee3d2d5c87)) +* **components:** Update supported large model reference names that can be resolved by function based component in _implementation/llm ([9ce2866](https://github.com/kubeflow/pipelines/commit/9ce28665276a74184339ee86c8ca84f8368fb8b4)) +* **components:** Upgrade LLM evaluation classification and text generation pipelines to preview ([b350ac4](https://github.com/kubeflow/pipelines/commit/b350ac4ddc32bd699c4cf92e3f6774088fb89f4f)) +* **components:** Upload tensorboard metrics from RLHF pipeline if a tensorboard resource id is provided ([2bb57ec](https://github.com/kubeflow/pipelines/commit/2bb57ece351757ab5aefa57c74fda397425abd00)) +* **components:** Use 64 v3 TPUs for llm pipelines ([45fe8e8](https://github.com/kubeflow/pipelines/commit/45fe8e86583646143d1685d9e04d887ff27440ee)) +* **components:** use GCPC project id placeholder as project parameter default ([88e1045](https://github.com/kubeflow/pipelines/commit/88e1045c116a6dc8adac83b5936821fe2ef9b263)) +* **components:** Use t5-xl reward model when tuning t5-xxl ([6468b4d](https://github.com/kubeflow/pipelines/commit/6468b4db11c2cd60a7b2dba7482ab170a129982d)) +* **deployment:** add option to deploy mysql in KFP standalone ([\#9855](https://github.com/kubeflow/pipelines/issues/9855)) ([b086020](https://github.com/kubeflow/pipelines/commit/b086020a249bd7c99ceaf54b6c8d4535f9f73df0)) +* **eval:** Implement embedding metrics importing ([b71d43e](https://github.com/kubeflow/pipelines/commit/b71d43eff3d4cf95b9b67eb3a890524f9d115807)) +* **manifests:** Add a postgresql deployment manifest in third-party folder ([\#9581](https://github.com/kubeflow/pipelines/issues/9581)) ([49bfda9](https://github.com/kubeflow/pipelines/commit/49bfda90cadc6437173909dea5b02cffc7cd7e66)) +* **mlmd:** Introduce PostgreSQL kustomization for MLMD. ([\#9927](https://github.com/kubeflow/pipelines/issues/9927)) ([b6be4ea](https://github.com/kubeflow/pipelines/commit/b6be4ea79bd4828e48f78eba5d69ef332524f309)) +* **sdk:** add logging at end of executor execution ([\#9895](https://github.com/kubeflow/pipelines/issues/9895)) ([ef0788d](https://github.com/kubeflow/pipelines/commit/ef0788d98690d0c70f747d8900ed719ce1328b35)) +* **sdk:** enable dependency-free runtime install of kfp ([\#9886](https://github.com/kubeflow/pipelines/issues/9886)) ([cf0e0cf](https://github.com/kubeflow/pipelines/commit/cf0e0cf87c7a33f906e2bc31da8c5356ed75a831)) +* **sdk:** support `PipelineTaskFinalStatus` in tasks that use `.ignore_upstream_failure()` ([\#10010](https://github.com/kubeflow/pipelines/issues/10010)) ([e99f270](https://github.com/kubeflow/pipelines/commit/e99f2704fc164039d9106a76223ee4abf9402bfb)) +* **sdk:** support dsl.If, dsl.Elif, and dsl.Else ([\#9894](https://github.com/kubeflow/pipelines/issues/9894)) ([c6b236d](https://github.com/kubeflow/pipelines/commit/c6b236d1a0a2385421e823512bd4c37041f1af26)) +* Adding new test infrastructure for e2e pipeline tests ([d98fa90](https://github.com/kubeflow/pipelines/commit/d98fa90bff79f3c13853d6ed9044c308253deba4)) + + +### Bug Fixes + +* **backend:** Fix performance issue within a mysql request ([\#9680](https://github.com/kubeflow/pipelines/issues/9680)) ([213dd5a](https://github.com/kubeflow/pipelines/commit/213dd5a1afc436ea207a466f69af3f6bd528b058)) +* **backend:** fix timeouts with list run api. Fixes [\#9780](https://github.com/kubeflow/pipelines/issues/9780) ([\#9806](https://github.com/kubeflow/pipelines/issues/9806)) ([a6af41c](https://github.com/kubeflow/pipelines/commit/a6af41c23be0fdc2a038c8b46725faa49e8909c1)) +* **backend:** Move ConMaxLifeTime back to DbConfig.ConMaxLifeTime. ([\#9873](https://github.com/kubeflow/pipelines/issues/9873)) ([fe60742](https://github.com/kubeflow/pipelines/commit/fe60742b000763b0d589d3124b544091a0aa29fb)) +* **backend:** OutPutPath dir creation mode Fixes [\#7629](https://github.com/kubeflow/pipelines/issues/7629) ([\#9946](https://github.com/kubeflow/pipelines/issues/9946)) ([4003e56](https://github.com/kubeflow/pipelines/commit/4003e562713bd04fa94387d8b53dfbe3cf31cb12)) +* **backend:** Sync scheduled workflows v1 if APIVersion and Kind are missing. Fixes [\#9809](https://github.com/kubeflow/pipelines/issues/9809) ([\#9968](https://github.com/kubeflow/pipelines/issues/9968)) ([dcaafee](https://github.com/kubeflow/pipelines/commit/dcaafeee8b98e2733444455e7117b628f017422d)) +* **backend:** update requirements scripts ([\#10009](https://github.com/kubeflow/pipelines/issues/10009)) ([434b41a](https://github.com/kubeflow/pipelines/commit/434b41a19c983432e5f1ba218ac29e5075604db9)) +* **components:** Disable caching for LLM pipeline components that store temporary artifacts ([4fd1c02](https://github.com/kubeflow/pipelines/commit/4fd1c02fc0a17d4d1272dde69d81c0bbb1fe18b4)) +* **components:** fix parent_model parameter of ModelUploadOp ignored ([5a0e2bd](https://github.com/kubeflow/pipelines/commit/5a0e2bdef086cdcb96c7a33ff6d883cd063cb375)) +* **components:** Fix proto reference from range to _range in model evaluation preview utils function ([f323acf](https://github.com/kubeflow/pipelines/commit/f323acf4eba80d9909fa23dfafff0ef8adcf05a9)) +* **components:** Fix the feature transform engine arguments ([42df5e1](https://github.com/kubeflow/pipelines/commit/42df5e1301de4300c2b84205ef22c786f791d4c3)) +* **components:** Have RLHF importer use default image if override is falsy ([3b8cea0](https://github.com/kubeflow/pipelines/commit/3b8cea060fc3088520666fea26e6452bda2fdb15)) +* **components:** include model version in upload model output artifact (fix) ([b0cccfe](https://github.com/kubeflow/pipelines/commit/b0cccfee9432d2e787ba1f74eb8beb906222bea8)) +* **components:** Minor update for chunking parameter name ([07156ae](https://github.com/kubeflow/pipelines/commit/07156ae8a6c59b378de3a8e960cd7c703130037a)) +* **components:** Move model eval version.py to _implementation folder ([7f23bfc](https://github.com/kubeflow/pipelines/commit/7f23bfc88813e54927e46f1b019ea86c0f03ce70)) +* **components:** Update package import for google protobuf ([2cfe463](https://github.com/kubeflow/pipelines/commit/2cfe4636ae362efff177329143feb6f7f6f6a8a5)) +* **frontend:** content is not available ([\#9720](https://github.com/kubeflow/pipelines/issues/9720)) ([e137ae7](https://github.com/kubeflow/pipelines/commit/e137ae7faccad207fedbeeff80f8502e49a1fbc5)) +* **frontend:** Introduce ALLOWED_ARTIFACT_DOMAIN_REGEX flag to prevent accessing undesired domains. Remove user input string from server response. ([\#9844](https://github.com/kubeflow/pipelines/issues/9844)) ([83d7e71](https://github.com/kubeflow/pipelines/commit/83d7e719d08c73c2c535722b66b77cdf0cb4cd08)) +* **frontend:** Missing pipeline version name in new run page. ([\#9799](https://github.com/kubeflow/pipelines/issues/9799)) ([0153430](https://github.com/kubeflow/pipelines/commit/0153430206567e5c50c878bc7b2fcdf0a79817c0)) +* **frontend:** Recurring run card in experiment details page ([\#9697](https://github.com/kubeflow/pipelines/issues/9697)) ([d1be1d9](https://github.com/kubeflow/pipelines/commit/d1be1d9ffdbf4a8bcf9aa8df36ec22ac182ceb7e)) +* **frontend:** Splitting logsDetails into lines based on CR and LF. Fixes [\#9593](https://github.com/kubeflow/pipelines/issues/9593) ([\#9594](https://github.com/kubeflow/pipelines/issues/9594)) ([536d93a](https://github.com/kubeflow/pipelines/commit/536d93a1bffe035bf5222e7a48faf59d3b053800)) +* **manifests:** Update persistence agent manifests in marketplace helm chart ([\#9908](https://github.com/kubeflow/pipelines/issues/9908)) ([0fed207](https://github.com/kubeflow/pipelines/commit/0fed207e40535825a74c8b228ad2e9fce87b0a71)) +* **samples:** Update execution_order, loop_output samples to v2 pipelines ([\#9867](https://github.com/kubeflow/pipelines/issues/9867)) ([3e3a747](https://github.com/kubeflow/pipelines/commit/3e3a747b582abf21d95149502343d8efdfa5fc62)) +* **samples:** Update loop_parameter, loop_static samples to v2 pipelines ([\#9870](https://github.com/kubeflow/pipelines/issues/9870)) ([ff2e002](https://github.com/kubeflow/pipelines/commit/ff2e002157472cd69eef74c2010756797e4ed460)) +* **samples:** update samples to v2 pipelines ([\#9851](https://github.com/kubeflow/pipelines/issues/9851)) ([1002e0c](https://github.com/kubeflow/pipelines/commit/1002e0cf8cbb452c8839d4631ce90851e293581c)) +* **samples:** Update volume_ops sample to v2 pipelines ([\#9877](https://github.com/kubeflow/pipelines/issues/9877)) ([90cec16](https://github.com/kubeflow/pipelines/commit/90cec167c0e49e115910928b00b5c5e50eaeed7c)) +* **sdk:** fix --no-deps flag usage ([\#9982](https://github.com/kubeflow/pipelines/issues/9982)) ([cc2cd58](https://github.com/kubeflow/pipelines/commit/cc2cd5891822ff841d4447dfd097764d26a2dda5)) +* **sdk:** fix click dependency bug ([\#9634](https://github.com/kubeflow/pipelines/issues/9634)) ([92c02a6](https://github.com/kubeflow/pipelines/commit/92c02a61723d4f19882ffcf6fd9f82a152a8a576)) +* **sdk:** fix GCPC break in KFP SDK ([\#9791](https://github.com/kubeflow/pipelines/issues/9791)) ([540294a](https://github.com/kubeflow/pipelines/commit/540294aedb9622b13063fdbee287411e68ba656a)) +* **sdk:** fix GitHub release script ([\#9663](https://github.com/kubeflow/pipelines/issues/9663)) ([e92d8bc](https://github.com/kubeflow/pipelines/commit/e92d8bc7228159cbec8c3ffbe51501a83ac99622)) +* **sdk:** fix incorrect sub-DAG output type when using `dsl.Collected` ([\#10069](https://github.com/kubeflow/pipelines/issues/10069)) ([fcdff29](https://github.com/kubeflow/pipelines/commit/fcdff294a6323f6cb1c0e574fc7aa5ccc25e420b)) +* Move stale GHA operation config to the right place ([\#9935](https://github.com/kubeflow/pipelines/issues/9935)) ([63a0803](https://github.com/kubeflow/pipelines/commit/63a0803e3a355b7cade2ddef69e7b57d96707436)) +* **sdk:** fix kfp sdk v2 readme ([\#9668](https://github.com/kubeflow/pipelines/issues/9668)) ([e5fe981](https://github.com/kubeflow/pipelines/commit/e5fe981c1af88b02122eeb1a46fead3a26993aeb)) +* **sdk:** various kfp-dsl fixes ([\#9785](https://github.com/kubeflow/pipelines/issues/9785)) ([8ad9716](https://github.com/kubeflow/pipelines/commit/8ad97167ff9ea589d396728fdec4413fd559ade1)) + + +### Other Pull Requests + +* Intel oneAPI XGBoost daal4py example pipeline ([\#10044](https://github.com/kubeflow/pipelines/issues/10044)) ([271d4eb](https://github.com/kubeflow/pipelines/commit/271d4ebfafa5a3fab7f100212fd14e1eb28421bd)) +* No public description ([adb8677](https://github.com/kubeflow/pipelines/commit/adb86777a0c8bf8c28bb0cee1d936daf70d9a59f)) +* fix(components):Update batch_prediction_*_gcs_source to predictions_*_gcs_source in information retrieval preprocessor ([e3bf085](https://github.com/kubeflow/pipelines/commit/e3bf085997aabc9024eed1fd2e002f77cc4fc43e)) +* Fix Persistence Agent SA Token time interval ([\#9892](https://github.com/kubeflow/pipelines/issues/9892)) ([6dfcee7](https://github.com/kubeflow/pipelines/commit/6dfcee7fa9d0e54a4797189d1f437367d4d9f4a2)) +* feat(backend) Enable auth between pesistence agent and pipelineAPI (ReportServer) ([\#9699](https://github.com/kubeflow/pipelines/issues/9699)) ([cb18d00](https://github.com/kubeflow/pipelines/commit/cb18d00bbbaed9cd77fc50dce739ed62c72b2356)) +* fix(backend) Replace LEFT with INNER JOIN when Archive Experiment ([\#9730](https://github.com/kubeflow/pipelines/issues/9730)) ([de89b1c](https://github.com/kubeflow/pipelines/commit/de89b1c6580d6efb69a4234d7d490ac24db9b3c9)) + ### [2.0.1](https://github.com/kubeflow/pipelines/compare/2.0.0...2.0.1) (2023-08-17) diff --git a/OWNERS b/OWNERS index 6ff925e7e3..857293fbdd 100644 --- a/OWNERS +++ b/OWNERS @@ -1,8 +1,15 @@ approvers: - - chensun - - IronPan - - james-jwu - - zijianjoy + - ckadner + - Tomcli + - fenglixa + - pugangxa + - scrapcodes + - yhwang reviewers: - - chensun - - zijianjoy + - ckadner + - Tomcli + - fenglixa + - pugangxa + - jinchihe + - scrapcodes + - yhwang diff --git a/VERSION b/VERSION index 10bf840ed5..50ffc5aa7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.1 \ No newline at end of file +2.0.3 diff --git a/backend/Dockerfile.driver b/backend/Dockerfile.driver index 78b2eb3133..5ffc60a4aa 100644 --- a/backend/Dockerfile.driver +++ b/backend/Dockerfile.driver @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.20.4-alpine3.17 as builder +FROM golang:1.20.9-alpine3.17 as builder WORKDIR /go/src/github.com/kubeflow/pipelines COPY . . diff --git a/backend/Dockerfile.launcher b/backend/Dockerfile.launcher index b936267ac9..4269ec52ef 100644 --- a/backend/Dockerfile.launcher +++ b/backend/Dockerfile.launcher @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.20.4-alpine3.17 as builder +FROM golang:1.20.9-alpine3.17 as builder WORKDIR /go/src/github.com/kubeflow/pipelines COPY . . diff --git a/backend/Dockerfile.visualization b/backend/Dockerfile.visualization index 4bcdcb03a0..f1e0d8d0d1 100644 --- a/backend/Dockerfile.visualization +++ b/backend/Dockerfile.visualization @@ -18,7 +18,7 @@ # and exporter.py files in the directory specified above. # This image should be in sync with image in backend/src/apiserver/visualization/update_requirements.sh. -FROM tensorflow/tensorflow:2.5.1 +FROM tensorflow/tensorflow:2.10.1 RUN apt-get update \ && apt-get install -y wget curl tar openssl diff --git a/backend/Makefile b/backend/Makefile index f5aab0d57e..91cda22c8b 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -5,7 +5,7 @@ CSV_PATH=backend/third_party_licenses # Whenever build command for any of the binaries change, we should update them both here and in backend/Dockerfiles. .PHONY: all -all: license_apiserver license_persistence_agent license_cache_server license_swf license_viewer +all: license_apiserver license_persistence_agent license_cache_server license_swf license_viewer license_driver license_launcher .PHONY: clean clean: @@ -21,6 +21,10 @@ $(BUILD)/swf: GO111MODULE=on go build -o $(BUILD)/swf github.com/kubeflow/pipelines/backend/src/crd/controller/scheduledworkflow $(BUILD)/viewer: GO111MODULE=on go build -o $(BUILD)/viewer github.com/kubeflow/pipelines/backend/src/crd/controller/viewer +$(BUILD)/driver: + GO111MODULE=on go build -o $(BUILD)/driver github.com/kubeflow/pipelines/backend/src/v2/cmd/driver +$(BUILD)/launcher: + GO111MODULE=on go build -o $(BUILD)/launcher github.com/kubeflow/pipelines/backend/src/v2/cmd/launcher-v2 # Update licenses info after dependencies changed. # See README.md#updating-licenses-info section for more details. @@ -39,6 +43,12 @@ license_swf: $(BUILD)/swf .PHONY: license_viewer license_viewer: $(BUILD)/viewer cd $(MOD_ROOT) && go-licenses csv ./backend/src/crd/controller/viewer > $(CSV_PATH)/viewer.csv +.PHONY: license_driver +license_driver: $(BUILD)/driver + cd $(MOD_ROOT) && go-licenses csv ./backend/src/v2/cmd/driver > $(CSV_PATH)/driver.csv +.PHONY: license_launcher +license_launcher: $(BUILD)/launcher + cd $(MOD_ROOT) && go-licenses csv ./backend/src/v2/cmd/launcher-v2 > $(CSV_PATH)/launcher.csv .PHONY: image_all image_all: image_apiserver image_persistence_agent image_cache image_swf image_viewer image_visualization diff --git a/backend/OWNERS b/backend/OWNERS index 5954ef1c39..6cbeef6afc 100644 --- a/backend/OWNERS +++ b/backend/OWNERS @@ -1,8 +1,6 @@ approvers: - - chensun - - gkcalat - - Linchin + - Tomcli + - yhwang reviewers: - - chensun - - gkcalat - - Linchin + - Tomcli + - yhwang diff --git a/backend/README.md b/backend/README.md index ecbad2f390..853d83dd6b 100644 --- a/backend/README.md +++ b/backend/README.md @@ -55,6 +55,13 @@ need to be regenerated and checked-in. Refer to [backend/api](./api/README.md) f make all ``` +## Updating python dependencies + +[pip-tools](https://github.com/jazzband/pip-tools) is used to manage python +dependencies. To update dependencies, edit [requirements.in](requirements.in) +and run `./update_requirements.sh` to update and pin the transitive +dependencies. + # Visualization Server Instructions ## Updating python dependencies diff --git a/backend/api/v1beta1/python_http_client/README.md b/backend/api/v1beta1/python_http_client/README.md index f0e94be6d2..1de30811ee 100644 --- a/backend/api/v1beta1/python_http_client/README.md +++ b/backend/api/v1beta1/python_http_client/README.md @@ -3,8 +3,8 @@ This file contains REST API specification for Kubeflow Pipelines. The file is au This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: -- API version: 2.0.1 -- Package version: 2.0.1 +- API version: 2.0.3 +- Package version: 2.0.3 - Build package: org.openapitools.codegen.languages.PythonClientCodegen For more information, please visit [https://www.google.com](https://www.google.com) diff --git a/backend/api/v1beta1/python_http_client/kfp_server_api/__init__.py b/backend/api/v1beta1/python_http_client/kfp_server_api/__init__.py index fc1497d659..86713cb581 100644 --- a/backend/api/v1beta1/python_http_client/kfp_server_api/__init__.py +++ b/backend/api/v1beta1/python_http_client/kfp_server_api/__init__.py @@ -14,7 +14,7 @@ from __future__ import absolute_import -__version__ = "2.0.1" +__version__ = "2.0.3" # import apis into sdk package from kfp_server_api.api.experiment_service_api import ExperimentServiceApi diff --git a/backend/api/v1beta1/python_http_client/kfp_server_api/api_client.py b/backend/api/v1beta1/python_http_client/kfp_server_api/api_client.py index 5b4cb571de..8a2be9ffd9 100644 --- a/backend/api/v1beta1/python_http_client/kfp_server_api/api_client.py +++ b/backend/api/v1beta1/python_http_client/kfp_server_api/api_client.py @@ -78,7 +78,7 @@ def __init__(self, configuration=None, header_name=None, header_value=None, self.default_headers[header_name] = header_value self.cookie = cookie # Set default User-Agent. - self.user_agent = 'OpenAPI-Generator/2.0.1/python' + self.user_agent = 'OpenAPI-Generator/2.0.3/python' self.client_side_validation = configuration.client_side_validation def __enter__(self): diff --git a/backend/api/v1beta1/python_http_client/kfp_server_api/configuration.py b/backend/api/v1beta1/python_http_client/kfp_server_api/configuration.py index fe73377512..fb157f5025 100644 --- a/backend/api/v1beta1/python_http_client/kfp_server_api/configuration.py +++ b/backend/api/v1beta1/python_http_client/kfp_server_api/configuration.py @@ -351,8 +351,8 @@ def to_debug_report(self): return "Python SDK Debug Report:\n"\ "OS: {env}\n"\ "Python Version: {pyversion}\n"\ - "Version of the API: 2.0.1\n"\ - "SDK Package Version: 2.0.1".\ + "Version of the API: 2.0.3\n"\ + "SDK Package Version: 2.0.3".\ format(env=sys.platform, pyversion=sys.version) def get_host_settings(self): diff --git a/backend/api/v1beta1/python_http_client/setup.py b/backend/api/v1beta1/python_http_client/setup.py index d3fd643008..9c9464f5ae 100644 --- a/backend/api/v1beta1/python_http_client/setup.py +++ b/backend/api/v1beta1/python_http_client/setup.py @@ -13,7 +13,7 @@ from setuptools import setup, find_packages # noqa: H301 NAME = "kfp-server-api" -VERSION = "2.0.1" +VERSION = "2.0.3" # To install the library, run the following # # python setup.py install diff --git a/backend/api/v1beta1/swagger/kfp_api_single_file.swagger.json b/backend/api/v1beta1/swagger/kfp_api_single_file.swagger.json index a4ebf3ca8a..9414f4cdcd 100644 --- a/backend/api/v1beta1/swagger/kfp_api_single_file.swagger.json +++ b/backend/api/v1beta1/swagger/kfp_api_single_file.swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "info": { "title": "Kubeflow Pipelines API", - "version": "2.0.1", + "version": "2.0.3", "description": "This file contains REST API specification for Kubeflow Pipelines. The file is autogenerated from the swagger definition.", "contact": { "name": "google", diff --git a/backend/api/v2beta1/python_http_client/README.md b/backend/api/v2beta1/python_http_client/README.md index dd98e9d6bd..be20533cf9 100644 --- a/backend/api/v2beta1/python_http_client/README.md +++ b/backend/api/v2beta1/python_http_client/README.md @@ -3,8 +3,8 @@ This file contains REST API specification for Kubeflow Pipelines. The file is au This Python package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project: -- API version: 2.0.1 -- Package version: 2.0.1 +- API version: 2.0.3 +- Package version: 2.0.3 - Build package: org.openapitools.codegen.languages.PythonClientCodegen For more information, please visit [https://www.google.com](https://www.google.com) diff --git a/backend/api/v2beta1/python_http_client/kfp_server_api/__init__.py b/backend/api/v2beta1/python_http_client/kfp_server_api/__init__.py index 87463f0a21..f7a521107d 100644 --- a/backend/api/v2beta1/python_http_client/kfp_server_api/__init__.py +++ b/backend/api/v2beta1/python_http_client/kfp_server_api/__init__.py @@ -14,7 +14,7 @@ from __future__ import absolute_import -__version__ = "2.0.1" +__version__ = "2.0.3" # import apis into sdk package from kfp_server_api.api.auth_service_api import AuthServiceApi diff --git a/backend/api/v2beta1/python_http_client/kfp_server_api/api_client.py b/backend/api/v2beta1/python_http_client/kfp_server_api/api_client.py index 5b4cb571de..8a2be9ffd9 100644 --- a/backend/api/v2beta1/python_http_client/kfp_server_api/api_client.py +++ b/backend/api/v2beta1/python_http_client/kfp_server_api/api_client.py @@ -78,7 +78,7 @@ def __init__(self, configuration=None, header_name=None, header_value=None, self.default_headers[header_name] = header_value self.cookie = cookie # Set default User-Agent. - self.user_agent = 'OpenAPI-Generator/2.0.1/python' + self.user_agent = 'OpenAPI-Generator/2.0.3/python' self.client_side_validation = configuration.client_side_validation def __enter__(self): diff --git a/backend/api/v2beta1/python_http_client/kfp_server_api/configuration.py b/backend/api/v2beta1/python_http_client/kfp_server_api/configuration.py index fe73377512..fb157f5025 100644 --- a/backend/api/v2beta1/python_http_client/kfp_server_api/configuration.py +++ b/backend/api/v2beta1/python_http_client/kfp_server_api/configuration.py @@ -351,8 +351,8 @@ def to_debug_report(self): return "Python SDK Debug Report:\n"\ "OS: {env}\n"\ "Python Version: {pyversion}\n"\ - "Version of the API: 2.0.1\n"\ - "SDK Package Version: 2.0.1".\ + "Version of the API: 2.0.3\n"\ + "SDK Package Version: 2.0.3".\ format(env=sys.platform, pyversion=sys.version) def get_host_settings(self): diff --git a/backend/api/v2beta1/python_http_client/setup.py b/backend/api/v2beta1/python_http_client/setup.py index d3fd643008..9c9464f5ae 100644 --- a/backend/api/v2beta1/python_http_client/setup.py +++ b/backend/api/v2beta1/python_http_client/setup.py @@ -13,7 +13,7 @@ from setuptools import setup, find_packages # noqa: H301 NAME = "kfp-server-api" -VERSION = "2.0.1" +VERSION = "2.0.3" # To install the library, run the following # # python setup.py install diff --git a/backend/api/v2beta1/swagger/kfp_api_single_file.swagger.json b/backend/api/v2beta1/swagger/kfp_api_single_file.swagger.json index bf218246d0..2a63b01dd6 100644 --- a/backend/api/v2beta1/swagger/kfp_api_single_file.swagger.json +++ b/backend/api/v2beta1/swagger/kfp_api_single_file.swagger.json @@ -2,7 +2,7 @@ "swagger": "2.0", "info": { "title": "Kubeflow Pipelines API", - "version": "2.0.1", + "version": "2.0.3", "description": "This file contains REST API specification for Kubeflow Pipelines. The file is autogenerated from the swagger definition.", "contact": { "name": "google", diff --git a/backend/metadata_writer/Dockerfile b/backend/metadata_writer/Dockerfile index 6111175f94..63394179ec 100644 --- a/backend/metadata_writer/Dockerfile +++ b/backend/metadata_writer/Dockerfile @@ -1,5 +1,5 @@ # ml-metadata package depends on tensorflow package -FROM python:3.7 +FROM python:3.8 COPY backend/metadata_writer/requirements.txt /kfp/metadata_writer/ RUN python3 -m pip install -r /kfp/metadata_writer/requirements.txt diff --git a/backend/metadata_writer/README.md b/backend/metadata_writer/README.md index 82cccca60f..eb8166edd6 100644 --- a/backend/metadata_writer/README.md +++ b/backend/metadata_writer/README.md @@ -4,6 +4,5 @@ [pip-tools](https://github.com/jazzband/pip-tools) is used to manage python dependencies. To update dependencies, edit [requirements.in](requirements.in) -and run `../update_requirements.sh python:3.7 requirements.txt` to update and pin the transitive +and run `./update_requirements.sh` to update and pin the transitive dependencies. - diff --git a/backend/metadata_writer/requirements.in b/backend/metadata_writer/requirements.in index 27130c336f..e45d3d7baf 100644 --- a/backend/metadata_writer/requirements.in +++ b/backend/metadata_writer/requirements.in @@ -1,3 +1,3 @@ kubernetes>=8.0.0,<11.0.0 -ml-metadata==1.5.0 +ml-metadata==1.14.0 lru-dict>=1.1.7,<2.0.0 diff --git a/backend/metadata_writer/requirements.txt b/backend/metadata_writer/requirements.txt index cc3c61470e..071621f3a8 100644 --- a/backend/metadata_writer/requirements.txt +++ b/backend/metadata_writer/requirements.txt @@ -1,32 +1,67 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=- - # -absl-py==0.12.0 # via ml-metadata -attrs==20.3.0 # via ml-metadata -cachetools==5.0.0 # via google-auth -certifi==2021.10.8 # via kubernetes, requests -charset-normalizer==2.0.10 # via requests -google-auth==2.4.1 # via kubernetes -grpcio==1.43.0 # via ml-metadata -idna==3.3 # via requests -kubernetes==10.1.0 # via -r - -lru-dict==1.1.7 # via -r - -ml-metadata==1.5.0 # via -r - -oauthlib==3.1.1 # via requests-oauthlib -protobuf==3.19.3 # via ml-metadata -pyasn1-modules==0.2.8 # via google-auth -pyasn1==0.4.8 # via pyasn1-modules, rsa -python-dateutil==2.8.2 # via kubernetes -pyyaml==3.13 # via kubernetes -requests-oauthlib==1.3.0 # via kubernetes -requests==2.27.1 # via kubernetes, requests-oauthlib -rsa==4.8 # via google-auth -six==1.16.0 # via absl-py, google-auth, grpcio, kubernetes, ml-metadata, python-dateutil -urllib3==1.26.8 # via kubernetes, requests -websocket-client==1.2.3 # via kubernetes +absl-py==1.4.0 + # via ml-metadata +attrs==21.4.0 + # via ml-metadata +cachetools==5.3.1 + # via google-auth +certifi==2023.7.22 + # via + # kubernetes + # requests +charset-normalizer==3.2.0 + # via requests +google-auth==2.23.0 + # via kubernetes +grpcio==1.58.0 + # via ml-metadata +idna==3.4 + # via requests +kubernetes==10.1.0 + # via -r - +lru-dict==1.2.0 + # via -r - +ml-metadata==1.14.0 + # via -r - +oauthlib==3.2.2 + # via requests-oauthlib +protobuf==3.20.3 + # via ml-metadata +pyasn1==0.5.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via google-auth +python-dateutil==2.8.2 + # via kubernetes +pyyaml==3.13 + # via kubernetes +requests==2.31.0 + # via + # kubernetes + # requests-oauthlib +requests-oauthlib==1.3.1 + # via kubernetes +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # kubernetes + # ml-metadata + # python-dateutil +urllib3==1.26.16 + # via + # google-auth + # kubernetes + # requests +websocket-client==1.6.3 + # via kubernetes # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/backend/metadata_writer/update_requirements.sh b/backend/metadata_writer/update_requirements.sh new file mode 100755 index 0000000000..f63fa4972d --- /dev/null +++ b/backend/metadata_writer/update_requirements.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# This image should be in sync with Dockerfile. +IMAGE="python:3.8" +../../hack/update-requirements.sh $IMAGE requirements.txt diff --git a/backend/requirements.txt b/backend/requirements.txt index 00191b2288..68cfc0f33f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,53 +1,55 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile --no-emit-index-url requirements.in +# pip-compile --output-file=- - # cachetools==5.3.1 # via google-auth -certifi==2023.5.7 +certifi==2023.7.22 # via # kfp-server-api # kubernetes # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.7 # via kfp docstring-parser==0.15 # via kfp -google-api-core==2.11.0 +google-api-core==2.11.1 # via # google-cloud-core # google-cloud-storage # kfp -google-auth==2.19.1 +google-auth==2.23.0 # via # google-api-core # google-cloud-core # google-cloud-storage # kfp # kubernetes -google-cloud-core==2.3.2 +google-cloud-core==2.3.3 # via google-cloud-storage -google-cloud-storage==2.9.0 +google-cloud-storage==2.11.0 # via kfp google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.5.0 +google-resumable-media==2.6.0 # via google-cloud-storage -googleapis-common-protos==1.59.0 +googleapis-common-protos==1.60.0 # via google-api-core idna==3.4 # via requests -kfp==2.0.0rc1 - # via -r requirements.in +importlib-metadata==6.7.0 + # via click +kfp==2.0.1 + # via -r - kfp-pipeline-spec==0.2.2 # via kfp -kfp-server-api==2.0.0-rc.1 +kfp-server-api==2.0.1 # via kfp -kubernetes==23.6.0 +kubernetes==26.1.0 # via kfp oauthlib==3.2.2 # via requests-oauthlib @@ -67,7 +69,7 @@ python-dateutil==2.8.2 # via # kfp-server-api # kubernetes -pyyaml==6.0 +pyyaml==6.0.1 # via # kfp # kubernetes @@ -86,12 +88,15 @@ rsa==4.9 # via google-auth six==1.16.0 # via - # google-auth # kfp-server-api # kubernetes # python-dateutil tabulate==0.9.0 # via kfp +typing-extensions==4.7.1 + # via + # importlib-metadata + # kfp urllib3==1.26.16 # via # google-auth @@ -99,8 +104,10 @@ urllib3==1.26.16 # kfp-server-api # kubernetes # requests -websocket-client==1.5.2 +websocket-client==1.6.1 # via kubernetes +zipp==3.15.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/backend/src/agent/persistence/client/fake_namespace.go b/backend/src/agent/persistence/client/fake_namespace.go deleted file mode 100644 index bbc8c8e022..0000000000 --- a/backend/src/agent/persistence/client/fake_namespace.go +++ /dev/null @@ -1,85 +0,0 @@ -package client - -import ( - "context" - "errors" - "github.com/golang/glog" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - corev1 "k8s.io/client-go/applyconfigurations/core/v1" -) - -type FakeNamespaceClient struct { - namespace string - user string -} - -func (f *FakeNamespaceClient) SetReturnValues(namespace string, user string) { - f.namespace = namespace - f.user = user -} - -func (f FakeNamespaceClient) Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Namespace, error) { - if f.namespace == name && len(f.user) != 0 { - ns := v1.Namespace{ObjectMeta: metav1.ObjectMeta{ - Namespace: f.namespace, - Annotations: map[string]string{ - "owner": f.user, - }, - }} - return &ns, nil - } - return nil, errors.New("failed to get namespace") -} - -func (f FakeNamespaceClient) Create(ctx context.Context, namespace *v1.Namespace, opts metav1.CreateOptions) (*v1.Namespace, error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) Update(ctx context.Context, namespace *v1.Namespace, opts metav1.UpdateOptions) (*v1.Namespace, error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) UpdateStatus(ctx context.Context, namespace *v1.Namespace, opts metav1.UpdateOptions) (*v1.Namespace, error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { - glog.Error("This fake method is not yet implemented.") - return nil -} - -func (f FakeNamespaceClient) List(ctx context.Context, opts metav1.ListOptions) (*v1.NamespaceList, error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Namespace, err error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) Apply(ctx context.Context, namespace *corev1.NamespaceApplyConfiguration, opts metav1.ApplyOptions) (result *v1.Namespace, err error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) ApplyStatus(ctx context.Context, namespace *corev1.NamespaceApplyConfiguration, opts metav1.ApplyOptions) (result *v1.Namespace, err error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} - -func (f FakeNamespaceClient) Finalize(ctx context.Context, item *v1.Namespace, opts metav1.UpdateOptions) (*v1.Namespace, error) { - glog.Error("This fake method is not yet implemented.") - return nil, nil -} diff --git a/backend/src/agent/persistence/client/kubernetes_core.go b/backend/src/agent/persistence/client/kubernetes_core.go deleted file mode 100644 index 25605ba88a..0000000000 --- a/backend/src/agent/persistence/client/kubernetes_core.go +++ /dev/null @@ -1,87 +0,0 @@ -package client - -import ( - "context" - "fmt" - "os" - "time" - - "github.com/cenkalti/backoff" - "github.com/golang/glog" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - v1 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/rest" - - "github.com/kubeflow/pipelines/backend/src/common/util" -) - -type KubernetesCoreInterface interface { - NamespaceClient() v1.NamespaceInterface - GetNamespaceOwner(namespace string) (string, error) -} - -type KubernetesCore struct { - coreV1Client v1.CoreV1Interface -} - -func (c *KubernetesCore) NamespaceClient() v1.NamespaceInterface { - return c.coreV1Client.Namespaces() -} - -func (c *KubernetesCore) GetNamespaceOwner(namespace string) (string, error) { - if os.Getenv("MULTIUSER") == "" || os.Getenv("MULTIUSER") == "false" { - return "", nil - } - ns, err := c.NamespaceClient().Get(context.Background(), namespace, metav1.GetOptions{}) - if err != nil { - return "", errors.Wrapf(err, "failed to get namespace '%v'", namespace) - } - owner, ok := ns.Annotations["owner"] - if !ok { - return "", errors.New(fmt.Sprintf("namespace '%v' has no owner in the annotations", namespace)) - } - return owner, nil -} - -func createKubernetesCore(clientParams util.ClientParameters) (KubernetesCoreInterface, error) { - clientSet, err := getKubernetesClientset(clientParams) - if err != nil { - return nil, err - } - return &KubernetesCore{clientSet.CoreV1()}, nil -} - -// CreateKubernetesCoreOrFatal creates a new client for the Kubernetes pod. -func CreateKubernetesCoreOrFatal(initConnectionTimeout time.Duration, clientParams util.ClientParameters) KubernetesCoreInterface { - var client KubernetesCoreInterface - var err error - var operation = func() error { - client, err = createKubernetesCore(clientParams) - return err - } - b := backoff.NewExponentialBackOff() - b.MaxElapsedTime = initConnectionTimeout - err = backoff.Retry(operation, b) - - if err != nil { - glog.Fatalf("Failed to create namespace client. Error: %v", err) - } - return client -} - -func getKubernetesClientset(clientParams util.ClientParameters) (*kubernetes.Clientset, error) { - restConfig, err := rest.InClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "Failed to initialize kubernetes client.") - } - restConfig.QPS = float32(clientParams.QPS) - restConfig.Burst = clientParams.Burst - - clientSet, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return nil, errors.Wrap(err, "Failed to initialize kubernetes client set.") - } - return clientSet, nil -} diff --git a/backend/src/agent/persistence/client/kubernetes_core_fake.go b/backend/src/agent/persistence/client/kubernetes_core_fake.go deleted file mode 100644 index 73fa0e34fe..0000000000 --- a/backend/src/agent/persistence/client/kubernetes_core_fake.go +++ /dev/null @@ -1,37 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "k8s.io/client-go/kubernetes/typed/core/v1" -) - -type KubernetesCoreFake struct { - coreV1ClientFake *FakeNamespaceClient -} - -func (c *KubernetesCoreFake) NamespaceClient() v1.NamespaceInterface { - return c.coreV1ClientFake -} - -func (c *KubernetesCoreFake) GetNamespaceOwner(namespace string) (string, error) { - ns, err := c.NamespaceClient().Get(context.Background(), namespace, metav1.GetOptions{}) - if err != nil { - return "", err - } - owner, ok := ns.Annotations["owner"] - if !ok { - return "", errors.New(fmt.Sprintf("namespace '%v' has no owner in the annotations", namespace)) - } - return owner, nil -} - -func NewKubernetesCoreFake() *KubernetesCoreFake { - return &KubernetesCoreFake{&FakeNamespaceClient{}} -} -func (c *KubernetesCoreFake) Set(namespaceToReturn string, userToReturn string) { - c.coreV1ClientFake.SetReturnValues(namespaceToReturn, userToReturn) -} diff --git a/backend/src/agent/persistence/client/pipeline_client.go b/backend/src/agent/persistence/client/pipeline_client.go index 8d0d775bff..58260ef62a 100644 --- a/backend/src/agent/persistence/client/pipeline_client.go +++ b/backend/src/agent/persistence/client/pipeline_client.go @@ -17,11 +17,9 @@ package client import ( "context" "fmt" - "os" "strings" "time" - "github.com/kubeflow/pipelines/backend/src/apiserver/common" "google.golang.org/grpc/metadata" api "github.com/kubeflow/pipelines/backend/api/v1beta1/go_client" @@ -39,8 +37,8 @@ const ( type PipelineClientInterface interface { ReportWorkflow(workflow util.ExecutionSpec) error ReportScheduledWorkflow(swf *util.ScheduledWorkflow) error - ReadArtifact(request *api.ReadArtifactRequest, user string) (*api.ReadArtifactResponse, error) - ReportRunMetrics(request *api.ReportRunMetricsRequest, user string) (*api.ReportRunMetricsResponse, error) + ReadArtifact(request *api.ReadArtifactRequest) (*api.ReadArtifactResponse, error) + ReportRunMetrics(request *api.ReportRunMetricsRequest) (*api.ReportRunMetricsResponse, error) } type PipelineClient struct { @@ -174,17 +172,26 @@ func (p *PipelineClient) ReportScheduledWorkflow(swf *util.ScheduledWorkflow) er // ReadArtifact reads artifact content from run service. If the artifact is not present, returns // nil response. -func (p *PipelineClient) ReadArtifact(request *api.ReadArtifactRequest, user string) (*api.ReadArtifactResponse, error) { +func (p *PipelineClient) ReadArtifact(request *api.ReadArtifactRequest) (*api.ReadArtifactResponse, error) { pctx := context.Background() - if user != "" { - pctx = metadata.AppendToOutgoingContext(pctx, getKubeflowUserIDHeader(), - getKubeflowUserIDPrefix()+user) - } + pctx = metadata.AppendToOutgoingContext(pctx, "Authorization", + "Bearer "+p.tokenRefresher.GetToken()) + ctx, cancel := context.WithTimeout(pctx, time.Minute) defer cancel() response, err := p.runServiceClient.ReadArtifactV1(ctx, request) if err != nil { + statusCode, _ := status.FromError(err) + if statusCode.Code() == codes.Unauthenticated && strings.Contains(err.Error(), "service account token has expired") { + // If unauthenticated because SA token is expired, re-read/refresh the token and try again + p.tokenRefresher.RefreshToken() + return nil, util.NewCustomError(err, util.CUSTOM_CODE_TRANSIENT, + "Error while reporting workflow resource (code: %v, message: %v): %v", + statusCode.Code(), + statusCode.Message(), + err.Error()) + } // TODO(hongyes): check NotFound error code before skip the error. return nil, nil } @@ -193,17 +200,26 @@ func (p *PipelineClient) ReadArtifact(request *api.ReadArtifactRequest, user str } // ReportRunMetrics reports run metrics to run service. -func (p *PipelineClient) ReportRunMetrics(request *api.ReportRunMetricsRequest, user string) (*api.ReportRunMetricsResponse, error) { +func (p *PipelineClient) ReportRunMetrics(request *api.ReportRunMetricsRequest) (*api.ReportRunMetricsResponse, error) { pctx := context.Background() - if user != "" { - pctx = metadata.AppendToOutgoingContext(pctx, getKubeflowUserIDHeader(), - getKubeflowUserIDPrefix()+user) - } + pctx = metadata.AppendToOutgoingContext(pctx, "Authorization", + "Bearer "+p.tokenRefresher.GetToken()) + ctx, cancel := context.WithTimeout(pctx, time.Minute) defer cancel() response, err := p.runServiceClient.ReportRunMetricsV1(ctx, request) if err != nil { + statusCode, _ := status.FromError(err) + if statusCode.Code() == codes.Unauthenticated && strings.Contains(err.Error(), "service account token has expired") { + // If unauthenticated because SA token is expired, re-read/refresh the token and try again + p.tokenRefresher.RefreshToken() + return nil, util.NewCustomError(err, util.CUSTOM_CODE_TRANSIENT, + "Error while reporting workflow resource (code: %v, message: %v): %v", + statusCode.Code(), + statusCode.Message(), + err.Error()) + } // This call should always succeed unless the run doesn't exist or server is broken. In // either cases, the job should retry at a later time. return nil, util.NewCustomError(err, util.CUSTOM_CODE_TRANSIENT, @@ -211,19 +227,3 @@ func (p *PipelineClient) ReportRunMetrics(request *api.ReportRunMetricsRequest, } return response, nil } - -// TODO use config file & viper and "github.com/kubeflow/pipelines/backend/src/apiserver/common.GetKubeflowUserIDHeader()" -func getKubeflowUserIDHeader() string { - if value, ok := os.LookupEnv(common.KubeflowUserIDHeader); ok { - return value - } - return common.GoogleIAPUserIdentityHeader -} - -// TODO use of viper & viper and "github.com/kubeflow/pipelines/backend/src/apiserver/common.GetKubeflowUserIDPrefix()" -func getKubeflowUserIDPrefix() string { - if value, ok := os.LookupEnv(common.KubeflowUserIDPrefix); ok { - return value - } - return common.GoogleIAPUserIdentityPrefix -} diff --git a/backend/src/agent/persistence/client/pipeline_client_fake.go b/backend/src/agent/persistence/client/pipeline_client_fake.go index 6b1ff3a03e..42e9bce25b 100644 --- a/backend/src/agent/persistence/client/pipeline_client_fake.go +++ b/backend/src/agent/persistence/client/pipeline_client_fake.go @@ -57,7 +57,7 @@ func (p *PipelineClientFake) ReportScheduledWorkflow(swf *util.ScheduledWorkflow return nil } -func (p *PipelineClientFake) ReadArtifact(request *api.ReadArtifactRequest, user string) (*api.ReadArtifactResponse, error) { +func (p *PipelineClientFake) ReadArtifact(request *api.ReadArtifactRequest) (*api.ReadArtifactResponse, error) { if p.err != nil { return nil, p.err } @@ -65,7 +65,7 @@ func (p *PipelineClientFake) ReadArtifact(request *api.ReadArtifactRequest, user return p.artifacts[request.String()], nil } -func (p *PipelineClientFake) ReportRunMetrics(request *api.ReportRunMetricsRequest, user string) (*api.ReportRunMetricsResponse, error) { +func (p *PipelineClientFake) ReportRunMetrics(request *api.ReportRunMetricsRequest) (*api.ReportRunMetricsResponse, error) { p.reportedMetricsRequest = request return p.reportMetricsResponseStub, p.reportMetricsErrorStub } diff --git a/backend/src/agent/persistence/client/scheduled_workflow_client.go b/backend/src/agent/persistence/client/scheduled_workflow_client.go index bf8b2acb1c..3fe4090d45 100644 --- a/backend/src/agent/persistence/client/scheduled_workflow_client.go +++ b/backend/src/agent/persistence/client/scheduled_workflow_client.go @@ -62,5 +62,11 @@ func (c *ScheduledWorkflowClient) Get(namespace string, name string) ( "Error retrieving scheduled workflow (%v) in namespace (%v): %v", name, namespace, err) } + // If the APIVersion and Kind are not set then set them to the default values. + if schedule.APIVersion == "" && schedule.Kind == "" { + schedule.Kind = util.SwfKind + schedule.APIVersion = util.ApiVersionV1 + } + return util.NewScheduledWorkflow(schedule), nil } diff --git a/backend/src/agent/persistence/main.go b/backend/src/agent/persistence/main.go index 60a6a7f06b..20ab4a791d 100644 --- a/backend/src/agent/persistence/main.go +++ b/backend/src/agent/persistence/main.go @@ -100,10 +100,6 @@ func main() { } else { swfInformerFactory = swfinformers.NewFilteredSharedInformerFactory(swfClient, time.Second*30, namespace, nil) } - k8sCoreClient := client.CreateKubernetesCoreOrFatal(DefaultConnectionTimeout, util.ClientParameters{ - QPS: clientQPS, - Burst: clientBurst, - }) tokenRefresher := client.NewTokenRefresher(time.Duration(saTokenRefreshIntervalInSecs)*time.Second, nil) err = tokenRefresher.StartTokenRefreshTicker() @@ -127,7 +123,6 @@ func main() { swfInformerFactory, execInformer, pipelineClient, - k8sCoreClient, util.NewRealTime()) go swfInformerFactory.Start(stopCh) @@ -159,4 +154,5 @@ func init() { // TODO use viper/config file instead. Sync `saTokenRefreshIntervalFlagName` with the value from manifest file by using ENV var. flag.Int64Var(&saTokenRefreshIntervalInSecs, saTokenRefreshIntervalFlagName, DefaultSATokenRefresherIntervalInSecs, "Persistence agent service account token read interval in seconds. "+ "Defines how often `/var/run/secrets/kubeflow/tokens/kubeflow-persistent_agent-api-token` to be read") + } diff --git a/backend/src/agent/persistence/persistence_agent.go b/backend/src/agent/persistence/persistence_agent.go index d234df09bf..d280b74dbd 100644 --- a/backend/src/agent/persistence/persistence_agent.go +++ b/backend/src/agent/persistence/persistence_agent.go @@ -46,7 +46,6 @@ func NewPersistenceAgent( swfInformerFactory swfinformers.SharedInformerFactory, execInformer util.ExecutionInformer, pipelineClient *client.PipelineClient, - k8sCoreClient client.KubernetesCoreInterface, time util.TimeInterface) *PersistenceAgent { // obtain references to shared informers swfInformer := swfInformerFactory.Scheduledworkflow().V1beta1().ScheduledWorkflows() @@ -63,7 +62,7 @@ func NewPersistenceAgent( workflowWorker := worker.NewPersistenceWorker(time, workflowregister.WorkflowKind, execInformer, true, - worker.NewWorkflowSaver(workflowClient, pipelineClient, k8sCoreClient, ttlSecondsAfterWorkflowFinish)) + worker.NewWorkflowSaver(workflowClient, pipelineClient, ttlSecondsAfterWorkflowFinish)) agent := &PersistenceAgent{ swfClient: swfClient, diff --git a/backend/src/agent/persistence/worker/metrics_reporter.go b/backend/src/agent/persistence/worker/metrics_reporter.go index 021ff970f5..c7a708cbf0 100644 --- a/backend/src/agent/persistence/worker/metrics_reporter.go +++ b/backend/src/agent/persistence/worker/metrics_reporter.go @@ -42,7 +42,7 @@ func NewMetricsReporter(pipelineClient client.PipelineClientInterface) *MetricsR } // ReportMetrics reports workflow metrics to pipeline server. -func (r MetricsReporter) ReportMetrics(workflow util.ExecutionSpec, user string) error { +func (r MetricsReporter) ReportMetrics(workflow util.ExecutionSpec) error { if !workflow.ExecutionStatus().HasMetrics() { return nil } @@ -52,14 +52,14 @@ func (r MetricsReporter) ReportMetrics(workflow util.ExecutionSpec, user string) // Skip reporting if the workflow doesn't have the run id label return nil } - runMetrics, partialFailures := workflow.ExecutionStatus().CollectionMetrics(r.pipelineClient.ReadArtifact, user) + runMetrics, partialFailures := workflow.ExecutionStatus().CollectionMetrics(r.pipelineClient.ReadArtifact) if len(runMetrics) == 0 { return aggregateErrors(partialFailures) } reportMetricsResponse, err := r.pipelineClient.ReportRunMetrics(&api.ReportRunMetricsRequest{ RunId: runID, Metrics: runMetrics, - }, user) + }) if err != nil { return err } diff --git a/backend/src/agent/persistence/worker/metrics_reporter_test.go b/backend/src/agent/persistence/worker/metrics_reporter_test.go index 7fa3ba000d..c2b43faf2c 100644 --- a/backend/src/agent/persistence/worker/metrics_reporter_test.go +++ b/backend/src/agent/persistence/worker/metrics_reporter_test.go @@ -32,11 +32,6 @@ import ( "k8s.io/apimachinery/pkg/types" ) -const ( - NamespaceName = "kf-namespace" - USER = "test-user@example.com" -) - func TestReportMetrics_NoCompletedNode_NoOP(t *testing.T) { pipelineFake := client.NewPipelineClientFake() @@ -57,7 +52,7 @@ func TestReportMetrics_NoCompletedNode_NoOP(t *testing.T) { }, }, }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.Nil(t, err) assert.Nil(t, pipelineFake.GetReportedMetricsRequest()) } @@ -82,7 +77,7 @@ func TestReportMetrics_NoRunID_NoOP(t *testing.T) { }, }, }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.Nil(t, err) assert.Nil(t, pipelineFake.GetReadArtifactRequest()) assert.Nil(t, pipelineFake.GetReportedMetricsRequest()) @@ -109,7 +104,7 @@ func TestReportMetrics_NoArtifact_NoOP(t *testing.T) { }, }, }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.Nil(t, err) assert.Nil(t, pipelineFake.GetReadArtifactRequest()) assert.Nil(t, pipelineFake.GetReportedMetricsRequest()) @@ -139,7 +134,7 @@ func TestReportMetrics_NoMetricsArtifact_NoOP(t *testing.T) { }, }, }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.Nil(t, err) assert.Nil(t, pipelineFake.GetReadArtifactRequest()) assert.Nil(t, pipelineFake.GetReportedMetricsRequest()) @@ -182,7 +177,7 @@ func TestReportMetrics_Succeed(t *testing.T) { Results: []*api.ReportRunMetricsResponse_ReportRunMetricResult{}, }, nil) - err1 := reporter.ReportMetrics(workflow, USER) + err1 := reporter.ReportMetrics(workflow) assert.Nil(t, err1) expectedMetricsRequest := &api.ReportRunMetricsRequest{ @@ -241,7 +236,7 @@ func TestReportMetrics_EmptyArchive_Fail(t *testing.T) { Data: []byte(artifactData), }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.NotNil(t, err) assert.True(t, util.HasCustomCode(err, util.CUSTOM_CODE_PERMANENT)) @@ -284,7 +279,7 @@ func TestReportMetrics_MultipleFilesInArchive_Fail(t *testing.T) { Data: []byte(artifactData), }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.NotNil(t, err) assert.True(t, util.HasCustomCode(err, util.CUSTOM_CODE_PERMANENT)) @@ -326,7 +321,7 @@ func TestReportMetrics_InvalidMetricsJSON_Fail(t *testing.T) { Data: []byte(artifactData), }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.NotNil(t, err) assert.True(t, util.HasCustomCode(err, util.CUSTOM_CODE_PERMANENT)) @@ -387,7 +382,7 @@ func TestReportMetrics_InvalidMetricsJSON_PartialFail(t *testing.T) { Data: []byte(validArtifactData), }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) // Partial failure is reported while valid metrics are reported. assert.NotNil(t, err) @@ -447,7 +442,7 @@ func TestReportMetrics_CorruptedArchiveFile_Fail(t *testing.T) { Data: []byte("invalid tgz content"), }) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.NotNil(t, err) assert.True(t, util.HasCustomCode(err, util.CUSTOM_CODE_PERMANENT)) @@ -511,7 +506,7 @@ func TestReportMetrics_MultiplMetricErrors_TransientErrowWin(t *testing.T) { }, }, nil) - err := reporter.ReportMetrics(workflow, USER) + err := reporter.ReportMetrics(workflow) assert.NotNil(t, err) assert.True(t, util.HasCustomCode(err, util.CUSTOM_CODE_TRANSIENT)) @@ -520,8 +515,6 @@ func TestReportMetrics_MultiplMetricErrors_TransientErrowWin(t *testing.T) { func TestReportMetrics_Unauthorized(t *testing.T) { pipelineFake := client.NewPipelineClientFake() reporter := NewMetricsReporter(pipelineFake) - k8sFake := client.NewKubernetesCoreFake() - k8sFake.Set(NamespaceName, USER) workflow := util.NewWorkflow(&workflowapi.Workflow{ ObjectMeta: metav1.ObjectMeta{ @@ -557,7 +550,7 @@ func TestReportMetrics_Unauthorized(t *testing.T) { Results: []*api.ReportRunMetricsResponse_ReportRunMetricResult{}, }, errors.New("failed to read artifacts")) - err1 := reporter.ReportMetrics(workflow, USER) + err1 := reporter.ReportMetrics(workflow) assert.NotNil(t, err1) assert.Contains(t, err1.Error(), "failed to read artifacts") diff --git a/backend/src/agent/persistence/worker/persistence_worker_test.go b/backend/src/agent/persistence/worker/persistence_worker_test.go index e29226d140..bde3ef7e4e 100644 --- a/backend/src/agent/persistence/worker/persistence_worker_test.go +++ b/backend/src/agent/persistence/worker/persistence_worker_test.go @@ -53,11 +53,9 @@ func TestPersistenceWorker_Success(t *testing.T) { // Set up pipeline client pipelineClient := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) // Set up peristence worker - saver := NewWorkflowSaver(workflowClient, pipelineClient, k8sClient, 100) + saver := NewWorkflowSaver(workflowClient, pipelineClient, 100) eventHandler := NewFakeEventHandler() worker := NewPersistenceWorker( util.NewFakeTimeForEpoch(), @@ -83,12 +81,11 @@ func TestPersistenceWorker_NotFoundError(t *testing.T) { }) workflowClient := client.NewWorkflowClientFake() - // Set up pipeline client and kubernetes client + // Set up pipeline client pipelineClient := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() // Set up peristence worker - saver := NewWorkflowSaver(workflowClient, pipelineClient, k8sClient, 100) + saver := NewWorkflowSaver(workflowClient, pipelineClient, 100) eventHandler := NewFakeEventHandler() worker := NewPersistenceWorker( util.NewFakeTimeForEpoch(), @@ -115,12 +112,11 @@ func TestPersistenceWorker_GetWorklowError(t *testing.T) { workflowClient := client.NewWorkflowClientFake() workflowClient.Put("MY_NAMESPACE", "MY_NAME", nil) - // Set up pipeline client and kubernetes client + // Set up pipeline client pipelineClient := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() // Set up peristence worker - saver := NewWorkflowSaver(workflowClient, pipelineClient, k8sClient, 100) + saver := NewWorkflowSaver(workflowClient, pipelineClient, 100) eventHandler := NewFakeEventHandler() worker := NewPersistenceWorker( util.NewFakeTimeForEpoch(), @@ -152,12 +148,9 @@ func TestPersistenceWorker_ReportWorkflowRetryableError(t *testing.T) { pipelineClient := client.NewPipelineClientFake() pipelineClient.SetError(util.NewCustomError(fmt.Errorf("Error"), util.CUSTOM_CODE_TRANSIENT, "My Retriable Error")) - //Set up kubernetes client - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) // Set up peristence worker - saver := NewWorkflowSaver(workflowClient, pipelineClient, k8sClient, 100) + saver := NewWorkflowSaver(workflowClient, pipelineClient, 100) eventHandler := NewFakeEventHandler() worker := NewPersistenceWorker( util.NewFakeTimeForEpoch(), @@ -188,11 +181,9 @@ func TestPersistenceWorker_ReportWorkflowNonRetryableError(t *testing.T) { pipelineClient := client.NewPipelineClientFake() pipelineClient.SetError(util.NewCustomError(fmt.Errorf("Error"), util.CUSTOM_CODE_PERMANENT, "My Permanent Error")) - // Set up kubernetes client - k8sClient := client.NewKubernetesCoreFake() // Set up peristence worker - saver := NewWorkflowSaver(workflowClient, pipelineClient, k8sClient, 100) + saver := NewWorkflowSaver(workflowClient, pipelineClient, 100) eventHandler := NewFakeEventHandler() worker := NewPersistenceWorker( util.NewFakeTimeForEpoch(), diff --git a/backend/src/agent/persistence/worker/workflow_saver.go b/backend/src/agent/persistence/worker/workflow_saver.go index 3b874273f1..5e93a60bb1 100644 --- a/backend/src/agent/persistence/worker/workflow_saver.go +++ b/backend/src/agent/persistence/worker/workflow_saver.go @@ -27,17 +27,15 @@ import ( type WorkflowSaver struct { client client.WorkflowClientInterface pipelineClient client.PipelineClientInterface - k8sClient client.KubernetesCoreInterface metricsReporter *MetricsReporter ttlSecondsAfterWorkflowFinish int64 } func NewWorkflowSaver(client client.WorkflowClientInterface, - pipelineClient client.PipelineClientInterface, k8sClient client.KubernetesCoreInterface, ttlSecondsAfterWorkflowFinish int64) *WorkflowSaver { + pipelineClient client.PipelineClientInterface, ttlSecondsAfterWorkflowFinish int64) *WorkflowSaver { return &WorkflowSaver{ client: client, pipelineClient: pipelineClient, - k8sClient: k8sClient, metricsReporter: NewMetricsReporter(pipelineClient), ttlSecondsAfterWorkflowFinish: ttlSecondsAfterWorkflowFinish, } @@ -70,11 +68,6 @@ func (s *WorkflowSaver) Save(key string, namespace string, name string, nowEpoch return nil } - user, err1 := s.k8sClient.GetNamespaceOwner(namespace) - if err1 != nil { - return util.Wrapf(err1, "Failed get '%v' namespace", namespace) - } - // Save this Workflow to the database. err = s.pipelineClient.ReportWorkflow(wf) retry := util.HasCustomCode(err, util.CUSTOM_CODE_TRANSIENT) @@ -94,5 +87,5 @@ func (s *WorkflowSaver) Save(key string, namespace string, name string, nowEpoch log.WithFields(log.Fields{ "Workflow": name, }).Infof("Syncing Workflow (%v): success, processing complete.", name) - return s.metricsReporter.ReportMetrics(wf, user) + return s.metricsReporter.ReportMetrics(wf) } diff --git a/backend/src/agent/persistence/worker/workflow_saver_test.go b/backend/src/agent/persistence/worker/workflow_saver_test.go index 10a16b7ccd..358f36600c 100644 --- a/backend/src/agent/persistence/worker/workflow_saver_test.go +++ b/backend/src/agent/persistence/worker/workflow_saver_test.go @@ -30,8 +30,6 @@ import ( func TestWorkflow_Save_Success(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) workflow := util.NewWorkflow(&workflowapi.Workflow{ ObjectMeta: metav1.ObjectMeta{ @@ -43,7 +41,7 @@ func TestWorkflow_Save_Success(t *testing.T) { workflowFake.Put("MY_NAMESPACE", "MY_NAME", workflow) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 100) err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) @@ -54,10 +52,8 @@ func TestWorkflow_Save_Success(t *testing.T) { func TestWorkflow_Save_NotFoundDuringGet(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 100) err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) @@ -69,12 +65,10 @@ func TestWorkflow_Save_NotFoundDuringGet(t *testing.T) { func TestWorkflow_Save_ErrorDuringGet(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) workflowFake.Put("MY_NAMESPACE", "MY_NAME", nil) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 100) err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) @@ -86,8 +80,6 @@ func TestWorkflow_Save_ErrorDuringGet(t *testing.T) { func TestWorkflow_Save_PermanentFailureWhileReporting(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) pipelineFake.SetError(util.NewCustomError(fmt.Errorf("Error"), util.CUSTOM_CODE_PERMANENT, "My Permanent Error")) @@ -102,7 +94,7 @@ func TestWorkflow_Save_PermanentFailureWhileReporting(t *testing.T) { workflowFake.Put("MY_NAMESPACE", "MY_NAME", workflow) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 100) err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) @@ -114,8 +106,6 @@ func TestWorkflow_Save_PermanentFailureWhileReporting(t *testing.T) { func TestWorkflow_Save_TransientFailureWhileReporting(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) pipelineFake.SetError(util.NewCustomError(fmt.Errorf("Error"), util.CUSTOM_CODE_TRANSIENT, "My Transient Error")) @@ -130,7 +120,7 @@ func TestWorkflow_Save_TransientFailureWhileReporting(t *testing.T) { workflowFake.Put("MY_NAMESPACE", "MY_NAME", workflow) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 100) err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) @@ -142,7 +132,6 @@ func TestWorkflow_Save_TransientFailureWhileReporting(t *testing.T) { func TestWorkflow_Save_SkippedDueToFinalStatue(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() // Add this will result in failure unless reporting is skipped pipelineFake.SetError(util.NewCustomError(fmt.Errorf("Error"), util.CUSTOM_CODE_PERMANENT, @@ -161,7 +150,7 @@ func TestWorkflow_Save_SkippedDueToFinalStatue(t *testing.T) { workflowFake.Put("MY_NAMESPACE", "MY_NAME", workflow) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 100) err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) @@ -172,8 +161,6 @@ func TestWorkflow_Save_SkippedDueToFinalStatue(t *testing.T) { func TestWorkflow_Save_FinalStatueNotSkippedDueToExceedTTL(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("MY_NAMESPACE", USER) // Add this will result in failure unless reporting is skipped pipelineFake.SetError(util.NewCustomError(fmt.Errorf("Error"), util.CUSTOM_CODE_PERMANENT, @@ -195,7 +182,7 @@ func TestWorkflow_Save_FinalStatueNotSkippedDueToExceedTTL(t *testing.T) { workflowFake.Put("MY_NAMESPACE", "MY_NAME", workflow) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 1) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 1) // Sleep 2 seconds to make sure workflow passed TTL time.Sleep(2 * time.Second) @@ -210,7 +197,6 @@ func TestWorkflow_Save_FinalStatueNotSkippedDueToExceedTTL(t *testing.T) { func TestWorkflow_Save_SkippedDDueToMissingRunID(t *testing.T) { workflowFake := client.NewWorkflowClientFake() pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() // Add this will result in failure unless reporting is skipped pipelineFake.SetError(util.NewCustomError(fmt.Errorf("Error"), util.CUSTOM_CODE_PERMANENT, @@ -225,33 +211,10 @@ func TestWorkflow_Save_SkippedDDueToMissingRunID(t *testing.T) { workflowFake.Put("MY_NAMESPACE", "MY_NAME", workflow) - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) + saver := NewWorkflowSaver(workflowFake, pipelineFake, 100) err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) assert.Equal(t, false, util.HasCustomCode(err, util.CUSTOM_CODE_TRANSIENT)) assert.Equal(t, nil, err) } - -func TestWorkflow_Save_FailedToGetUser(t *testing.T) { - workflowFake := client.NewWorkflowClientFake() - pipelineFake := client.NewPipelineClientFake() - k8sClient := client.NewKubernetesCoreFake() - k8sClient.Set("ORIGINAL_NAMESPACE", USER) - - workflow := util.NewWorkflow(&workflowapi.Workflow{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "MY_NAMESPACE", - Name: "MY_NAME", - Labels: map[string]string{util.LabelKeyWorkflowRunId: "MY_UUID"}, - }, - }) - - workflowFake.Put("MY_NAMESPACE", "MY_NAME", workflow) - - saver := NewWorkflowSaver(workflowFake, pipelineFake, k8sClient, 100) - - err := saver.Save("MY_KEY", "MY_NAMESPACE", "MY_NAME", 20) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), fmt.Sprintf("Failed get '%v' namespace", "MY_NAMESPACE")) -} diff --git a/backend/src/apiserver/client/sql.go b/backend/src/apiserver/client/sql.go index c4d44d61ae..026ef05619 100644 --- a/backend/src/apiserver/client/sql.go +++ b/backend/src/apiserver/client/sql.go @@ -15,13 +15,22 @@ package client import ( + "bytes" "fmt" "github.com/go-sql-driver/mysql" ) -func CreateMySQLConfig(user, password string, mysqlServiceHost string, - mysqlServicePort string, dbName string, mysqlGroupConcatMaxLen string, mysqlExtraParams map[string]string, +const ( + MYSQL_TEXT_FORMAT string = "longtext not null" + MYSQL_EXIST_ERROR string = "database exists" + + PGX_TEXT_FORMAT string = "text" + PGX_EXIST_ERROR string = "already exists" +) + +func CreateMySQLConfig(user, password, mysqlServiceHost, mysqlServicePort, + dbName, mysqlGroupConcatMaxLen string, mysqlExtraParams map[string]string, ) *mysql.Config { params := map[string]string{ "charset": "utf8", @@ -44,3 +53,26 @@ func CreateMySQLConfig(user, password string, mysqlServiceHost string, AllowNativePasswords: true, } } + +func CreatePostgreSQLConfig(user, password, postgresHost, dbName string, postgresPort uint16, +) string { + var b bytes.Buffer + if dbName != "" { + fmt.Fprintf(&b, "database=%s ", dbName) + } + if user != "" { + fmt.Fprintf(&b, "user=%s ", user) + } + if password != "" { + fmt.Fprintf(&b, "password=%s ", password) + } + if postgresHost != "" { + fmt.Fprintf(&b, "host=%s ", postgresHost) + } + if postgresPort != 0 { + fmt.Fprintf(&b, "port=%d ", postgresPort) + } + fmt.Fprint(&b, "sslmode=disable") + + return b.String() +} diff --git a/backend/src/apiserver/client_manager.go b/backend/src/apiserver/client_manager/client_manager.go similarity index 63% rename from backend/src/apiserver/client_manager.go rename to backend/src/apiserver/client_manager/client_manager.go index e1056697e5..261aef5605 100644 --- a/backend/src/apiserver/client_manager.go +++ b/backend/src/apiserver/client_manager/client_manager.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Kubeflow Authors +// Copyright 2018-2023 The Kubeflow Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package clientmanager import ( "database/sql" "fmt" "os" + "strings" "time" "github.com/cenkalti/backoff" + "github.com/go-sql-driver/mysql" "github.com/golang/glog" "github.com/jinzhu/gorm" _ "github.com/jinzhu/gorm/dialects/sqlite" @@ -35,25 +37,33 @@ import ( ) const ( - minioServiceHost = "MINIO_SERVICE_SERVICE_HOST" - minioServicePort = "MINIO_SERVICE_SERVICE_PORT" - minioServiceRegion = "MINIO_SERVICE_REGION" - minioServiceSecure = "MINIO_SERVICE_SECURE" - pipelineBucketName = "MINIO_PIPELINE_BUCKET_NAME" - pipelinePath = "MINIO_PIPELINE_PATH" - mysqlServiceHost = "DBConfig.Host" - mysqlServicePort = "DBConfig.Port" - mysqlUser = "DBConfig.User" - mysqlPassword = "DBConfig.Password" - mysqlDBName = "DBConfig.DBName" - mysqlGroupConcatMaxLen = "DBConfig.GroupConcatMaxLen" - mysqlExtraParams = "DBConfig.ExtraParams" - archiveLogFileName = "ARCHIVE_CONFIG_LOG_FILE_NAME" - archiveLogPathPrefix = "ARCHIVE_CONFIG_LOG_PATH_PREFIX" - dbConMaxLifeTime = "DBConfig.ConMaxLifeTime" - - visualizationServiceHost = "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST" - visualizationServicePort = "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT" + minioServiceHost = "MINIO_SERVICE_SERVICE_HOST" + minioServicePort = "MINIO_SERVICE_SERVICE_PORT" + minioServiceRegion = "MINIO_SERVICE_REGION" + minioServiceSecure = "MINIO_SERVICE_SECURE" + pipelineBucketName = "MINIO_PIPELINE_BUCKET_NAME" + pipelinePath = "MINIO_PIPELINE_PATH" + + mysqlServiceHost = "DBConfig.MySQLConfig.Host" + mysqlServicePort = "DBConfig.MySQLConfig.Port" + mysqlUser = "DBConfig.MySQLConfig.User" + mysqlPassword = "DBConfig.MySQLConfig.Password" + mysqlDBName = "DBConfig.MySQLConfig.DBName" + mysqlGroupConcatMaxLen = "DBConfig.MySQLConfig.GroupConcatMaxLen" + mysqlExtraParams = "DBConfig.MySQLConfig.ExtraParams" + + postgresHost = "DBConfig.PostgreSQLConfig.Host" + postgresPort = "DBConfig.PostgreSQLConfig.Port" + postgresUser = "DBConfig.PostgreSQLConfig.User" + postgresPassword = "DBConfig.PostgreSQLConfig.Password" + postgresDBName = "DBConfig.PostgreSQLConfig.DBName" + + archiveLogFileName = "ARCHIVE_CONFIG_LOG_FILE_NAME" + archiveLogPathPrefix = "ARCHIVE_CONFIG_LOG_PATH_PREFIX" + dbConMaxLifeTime = "DBConfig.ConMaxLifeTime" + + VisualizationServiceHost = "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST" + VisualizationServicePort = "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT" initConnectionTimeout = "InitConnectionTimeout" @@ -158,7 +168,7 @@ func (c *ClientManager) Authenticators() []auth.Authenticator { func (c *ClientManager) init() { glog.Info("Initializing client manager") - db := initDBClient(common.GetDurationConfig(initConnectionTimeout)) + db := InitDBClient(common.GetDurationConfig(initConnectionTimeout)) db.SetConnMaxLifetime(common.GetDurationConfig(dbConMaxLifeTime)) // time @@ -208,16 +218,12 @@ func (c *ClientManager) Close() { c.db.Close() } -func initDBClient(initConnectionTimeout time.Duration) *storage.DB { - driverName := common.GetStringConfig("DBConfig.DriverName") - var arg string - - switch driverName { - case "mysql": - arg = initMysql(driverName, initConnectionTimeout) - default: - glog.Fatalf("Driver %v is not supported", driverName) - } +func InitDBClient(initConnectionTimeout time.Duration) *storage.DB { + // Allowed driverName values: + // 1) To use MySQL, use `mysql` + // 2) To use PostgreSQL, use `pgx` + driverName := common.GetStringConfig("DBDriverName") + arg := initDBDriver(driverName, initConnectionTimeout) // db is safe for concurrent use by multiple goroutines // and maintains its own pool of idle connections. @@ -250,8 +256,18 @@ func initDBClient(initConnectionTimeout time.Duration) *storage.DB { &model.ResourceReference{}, ) - if response.Error != nil { - glog.Fatalf("Failed to initialize the databases.") + if ignoreAlreadyExistError(driverName, response.Error) != nil { + glog.Fatalf("Failed to initialize the databases. Error: %s", response.Error) + } + + var textFormat string + switch driverName { + case "mysql": + textFormat = client.MYSQL_TEXT_FORMAT + case "pgx": + textFormat = client.PGX_TEXT_FORMAT + default: + glog.Fatalf("Unsupported database driver %s, please use `mysql` for MySQL, or `pgx` for PostgreSQL.", driverName) } response = db.Model(&model.Experiment{}).RemoveIndex("Name") @@ -264,50 +280,71 @@ func initDBClient(initConnectionTimeout time.Duration) *storage.DB { glog.Fatalf("Failed to drop unique key on pipeline name. Error: %s", response.Error) } - response = db.Model(&model.ResourceReference{}).ModifyColumn("Payload", "longtext not null") + response = db.Model(&model.ResourceReference{}).ModifyColumn("Payload", textFormat) if response.Error != nil { glog.Fatalf("Failed to update the resource reference payload type. Error: %s", response.Error) } response = db.Model(&model.Run{}).AddIndex("experimentuuid_createatinsec", "ExperimentUUID", "CreatedAtInSec") - if response.Error != nil { + if ignoreAlreadyExistError(driverName, response.Error) != nil { glog.Fatalf("Failed to create index experimentuuid_createatinsec on run_details. Error: %s", response.Error) } response = db.Model(&model.Run{}).AddIndex("experimentuuid_conditions_finishedatinsec", "ExperimentUUID", "Conditions", "FinishedAtInSec") - if response.Error != nil { + if ignoreAlreadyExistError(driverName, response.Error) != nil { glog.Fatalf("Failed to create index experimentuuid_conditions_finishedatinsec on run_details. Error: %s", response.Error) } response = db.Model(&model.Run{}).AddIndex("namespace_createatinsec", "Namespace", "CreatedAtInSec") - if response.Error != nil { + if ignoreAlreadyExistError(driverName, response.Error) != nil { glog.Fatalf("Failed to create index namespace_createatinsec on run_details. Error: %s", response.Error) } response = db.Model(&model.Run{}).AddIndex("namespace_conditions_finishedatinsec", "Namespace", "Conditions", "FinishedAtInSec") - if response.Error != nil { + if ignoreAlreadyExistError(driverName, response.Error) != nil { glog.Fatalf("Failed to create index namespace_conditions_finishedatinsec on run_details. Error: %s", response.Error) } response = db.Model(&model.Pipeline{}).AddUniqueIndex("name_namespace_index", "Name", "Namespace") - if response.Error != nil { + if ignoreAlreadyExistError(driverName, response.Error) != nil { glog.Fatalf("Failed to create index name_namespace_index on run_details. Error: %s", response.Error) } - response = db.Model(&model.RunMetric{}). - AddForeignKey("RunUUID", "run_details(UUID)", "CASCADE" /* onDelete */, "CASCADE" /* update */) - if response.Error != nil { - glog.Fatalf("Failed to create a foreign key for RunID in run_metrics table. Error: %s", response.Error) - } - response = db.Model(&model.PipelineVersion{}). - AddForeignKey("PipelineId", "pipelines(UUID)", "CASCADE" /* onDelete */, "CASCADE" /* update */) - if response.Error != nil { - glog.Fatalf("Failed to create a foreign key for PipelineId in pipeline_versions table. Error: %s", response.Error) - } - response = db.Model(&model.Task{}). - AddForeignKey("RunUUID", "run_details(UUID)", "CASCADE" /* onDelete */, "CASCADE" /* update */) - if response.Error != nil { - glog.Fatalf("Failed to create a foreign key for RunUUID in task table. Error: %s", response.Error) + switch driverName { + case "pgx": + response = db.Model(&model.RunMetric{}). + AddForeignKey("\"RunUUID\"", "run_details(\"UUID\")", "CASCADE" /* onDelete */, "CASCADE" /* onUpdate */) + if ignoreAlreadyExistError(driverName, response.Error) != nil { + glog.Fatalf("Failed to create a foreign key for RunUUID in run_metrics table. Error: %s", response.Error) + } + response = db.Model(&model.PipelineVersion{}). + AddForeignKey("\"PipelineId\"", "pipelines(\"UUID\")", "CASCADE" /* onDelete */, "CASCADE" /* onUpdate */) + if ignoreAlreadyExistError(driverName, response.Error) != nil { + glog.Fatalf("Failed to create a foreign key for PipelineId in pipeline_versions table. Error: %s", response.Error) + } + response = db.Model(&model.Task{}). + AddForeignKey("\"RunUUID\"", "run_details(\"UUID\")", "CASCADE" /* onDelete */, "CASCADE" /* onUpdate */) + if ignoreAlreadyExistError(driverName, response.Error) != nil { + glog.Fatalf("Failed to create a foreign key for RunUUID in task table. Error: %s", response.Error) + } + case "mysql": + response = db.Model(&model.RunMetric{}). + AddForeignKey("RunUUID", "run_details(UUID)", "CASCADE" /* onDelete */, "CASCADE" /* onUpdate */) + if ignoreAlreadyExistError(driverName, response.Error) != nil { + glog.Fatalf("Failed to create a foreign key for RunUUID in run_metrics table. Error: %s", response.Error) + } + response = db.Model(&model.PipelineVersion{}). + AddForeignKey("PipelineId", "pipelines(UUID)", "CASCADE" /* onDelete */, "CASCADE" /* onUpdate */) + if ignoreAlreadyExistError(driverName, response.Error) != nil { + glog.Fatalf("Failed to create a foreign key for PipelineId in pipeline_versions table. Error: %s", response.Error) + } + response = db.Model(&model.Task{}). + AddForeignKey("RunUUID", "run_details(UUID)", "CASCADE" /* onDelete */, "CASCADE" /* onUpdate */) + if ignoreAlreadyExistError(driverName, response.Error) != nil { + glog.Fatalf("Failed to create a foreign key for RunUUID in task table. Error: %s", response.Error) + } + default: + glog.Fatalf("Driver %v is not supported, use \"mysql\" for MySQL, or \"pgx\" for PostgreSQL", driverName) } // Data backfill for pipeline_versions if this is the first time for @@ -320,44 +357,66 @@ func initDBClient(initConnectionTimeout time.Duration) *storage.DB { glog.Fatalf("Failed to backfill experiment UUID in run_details table: %s", err) } - response = db.Model(&model.Pipeline{}).ModifyColumn("Description", "longtext not null") + response = db.Model(&model.Pipeline{}).ModifyColumn("Description", textFormat) if response.Error != nil { glog.Fatalf("Failed to update pipeline description type. Error: %s", response.Error) } - // If the old unique index idx_pipeline_version_uuid_name on pipeline_versions exists, remove it. - rows, err := db.Raw(`show index from pipeline_versions where Key_name='idx_pipeline_version_uuid_name'`).Rows() - if err != nil { - glog.Fatalf("Failed to query pipeline_version table's indices. Error: %s", err) - } - if err := rows.Err(); err != nil { - glog.Fatalf("Failed to query pipeline_version table's indices. Error: %s", err) - } - if rows.Next() { - db.Exec(`drop index idx_pipeline_version_uuid_name on pipeline_versions`) + // Because PostgreSQL was supported later, there's no need to delete the relic index + if driverName == "mysql" { + // If the old unique index idx_pipeline_version_uuid_name on pipeline_versions exists, remove it. + rows, err := db.Raw(`show index from pipeline_versions where Key_name='idx_pipeline_version_uuid_name'`).Rows() + if err != nil { + glog.Fatalf("Failed to query pipeline_version table's indices. Error: %s", err) + } + if err := rows.Err(); err != nil { + glog.Fatalf("Failed to query pipeline_version table's indices. Error: %s", err) + } + if rows.Next() { + db.Exec(`drop index idx_pipeline_version_uuid_name on pipeline_versions`) + } + defer rows.Close() } - defer rows.Close() return storage.NewDB(db.DB(), storage.NewMySQLDialect()) } -// Initialize the connection string for connecting to Mysql database -// Format would be something like root@tcp(ip:port)/dbname?charset=utf8&loc=Local&parseTime=True. -func initMysql(driverName string, initConnectionTimeout time.Duration) string { - mysqlConfig := client.CreateMySQLConfig( - common.GetStringConfigWithDefault(mysqlUser, "root"), - common.GetStringConfigWithDefault(mysqlPassword, ""), - common.GetStringConfigWithDefault(mysqlServiceHost, "mysql"), - common.GetStringConfigWithDefault(mysqlServicePort, "3306"), - "", - common.GetStringConfigWithDefault(mysqlGroupConcatMaxLen, "1024"), - common.GetMapConfig(mysqlExtraParams), - ) +// Initializes Database driver. Use `driverName` to indicate which type of DB to use: +// 1) "mysql" for MySQL +// 2) "pgx" for PostgreSQL +func initDBDriver(driverName string, initConnectionTimeout time.Duration) string { + var sqlConfig, dbName string + var mysqlConfig *mysql.Config + switch driverName { + case "mysql": + mysqlConfig = client.CreateMySQLConfig( + common.GetStringConfigWithDefault(mysqlUser, "root"), + common.GetStringConfigWithDefault(mysqlPassword, ""), + common.GetStringConfigWithDefault(mysqlServiceHost, "mysql"), + common.GetStringConfigWithDefault(mysqlServicePort, "3306"), + "", + common.GetStringConfigWithDefault(mysqlGroupConcatMaxLen, "1024"), + common.GetMapConfig(mysqlExtraParams), + ) + sqlConfig = mysqlConfig.FormatDSN() + dbName = common.GetStringConfig(mysqlDBName) + case "pgx": + sqlConfig = client.CreatePostgreSQLConfig( + common.GetStringConfigWithDefault(postgresUser, "user"), + common.GetStringConfigWithDefault(postgresPassword, "password"), + common.GetStringConfigWithDefault(postgresHost, "postgresql"), + "postgres", + uint16(common.GetIntConfigWithDefault(postgresPort, 5432)), + ) + dbName = common.GetStringConfig(postgresDBName) + default: + glog.Fatalf("Driver %v is not supported, use \"mysql\" for MySQL, or \"pgx\" for PostgreSQL", driverName) + } var db *sql.DB var err error operation := func() error { - db, err = sql.Open(driverName, mysqlConfig.FormatDSN()) + db, err = sql.Open(driverName, sqlConfig) if err != nil { return err } @@ -365,7 +424,6 @@ func initMysql(driverName string, initConnectionTimeout time.Duration) string { } b := backoff.NewExponentialBackOff() b.MaxElapsedTime = initConnectionTimeout - // err = backoff.Retry(operation, b) err = backoff.RetryNotify(operation, b, func(e error, duration time.Duration) { glog.Errorf("%v", e) }) @@ -374,10 +432,9 @@ func initMysql(driverName string, initConnectionTimeout time.Duration) string { util.TerminateIfError(err) // Create database if not exist - dbName := common.GetStringConfig(mysqlDBName) operation = func() error { - _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", dbName)) - if err != nil { + _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", dbName)) + if ignoreAlreadyExistError(driverName, err) != nil { return err } return nil @@ -387,13 +444,30 @@ func initMysql(driverName string, initConnectionTimeout time.Duration) string { err = backoff.Retry(operation, b) util.TerminateIfError(err) - mysqlConfig.DBName = dbName - // When updating, return rows matched instead of rows affected. This counts rows that are being - // set as the same values as before. If updating using a primary key and rows matched is 0, then - // it means this row is not found. - // Config reference: https://github.com/go-sql-driver/mysql#clientfoundrows - mysqlConfig.ClientFoundRows = true - return mysqlConfig.FormatDSN() + + switch driverName { + case "mysql": + mysqlConfig.DBName = dbName + // When updating, return rows matched instead of rows affected. This counts rows that are being + // set as the same values as before. If updating using a primary key and rows matched is 0, then + // it means this row is not found. + // Config reference: https://github.com/go-sql-driver/mysql#clientfoundrows + mysqlConfig.ClientFoundRows = true + sqlConfig = mysqlConfig.FormatDSN() + case "pgx": + // Note: postgreSQL does not have the option `ClientFoundRows` + // Config reference: https://www.postgresql.org/docs/current/libpq-connect.html + sqlConfig = client.CreatePostgreSQLConfig( + common.GetStringConfigWithDefault(postgresUser, "root"), + common.GetStringConfigWithDefault(postgresPassword, ""), + common.GetStringConfigWithDefault(postgresHost, "postgresql"), + dbName, + uint16(common.GetIntConfigWithDefault(postgresPort, 5432)), + ) + default: + glog.Fatalf("Driver %v is not supported, use \"mysql\" for MySQL, or \"pgx\" for PostgreSQL", driverName) + } + return sqlConfig } func initMinioClient(initConnectionTimeout time.Duration) storage.ObjectStoreInterface { @@ -448,8 +522,8 @@ func initLogArchive() (logArchive archive.LogArchiveInterface) { return } -// newClientManager creates and Init a new instance of ClientManager. -func newClientManager() ClientManager { +// NewClientManager creates and Init a new instance of ClientManager. +func NewClientManager() ClientManager { clientManager := ClientManager{} clientManager.init() @@ -489,7 +563,7 @@ func initPipelineVersionsFromPipelines(db *gorm.DB) { func backfillExperimentIDToRunTable(db *gorm.DB) error { // check if there is any row in the run table has experiment ID being empty - rows, err := db.CommonDB().Query(`SELECT ExperimentUUID FROM run_details WHERE ExperimentUUID = '' LIMIT 1`) + rows, err := db.CommonDB().Query("SELECT \"ExperimentUUID\" FROM run_details WHERE \"ExperimentUUID\" = '' LIMIT 1") if err != nil { return err } @@ -516,3 +590,15 @@ func backfillExperimentIDToRunTable(db *gorm.DB) error { `) return err } + +// Returns the same error, if it's not "already exists" related. +// Otherwise, return nil. +func ignoreAlreadyExistError(driverName string, err error) error { + if driverName == "pgx" && err != nil && strings.Contains(err.Error(), client.PGX_EXIST_ERROR) { + return nil + } + if driverName == "mysql" && err != nil && strings.Contains(err.Error(), client.MYSQL_EXIST_ERROR) { + return nil + } + return err +} diff --git a/backend/src/apiserver/config/config.json b/backend/src/apiserver/config/config.json index e27bab65f6..aa7088ce77 100644 --- a/backend/src/apiserver/config/config.json +++ b/backend/src/apiserver/config/config.json @@ -1,9 +1,13 @@ { "DBConfig": { - "DriverName": "mysql", - "DataSourceName": "", - "DBName": "mlpipeline", - "GroupConcatMaxLen": "4194304", + "MySQLConfig": { + "DataSourceName": "", + "DBName": "mlpipeline", + "GroupConcatMaxLen": "4194304" + }, + "PostgreSQLConfig": { + "DBName": "mlpipeline" + }, "ConMaxLifeTime": "120s" }, "ObjectStoreConfig": { @@ -12,6 +16,7 @@ "BucketName": "mlpipeline", "PipelinePath": "pipelines" }, + "DBDriverName": "mysql", "ARCHIVE_CONFIG_LOG_FILE_NAME": "main.log", "ARCHIVE_CONFIG_LOG_PATH_PREFIX": "/artifacts", "InitConnectionTimeout": "6m", diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 802ef8a7cb..599c0694b7 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -35,6 +35,7 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/runtime" apiv1beta1 "github.com/kubeflow/pipelines/backend/api/v1beta1/go_client" apiv2beta1 "github.com/kubeflow/pipelines/backend/api/v2beta1/go_client" + cm "github.com/kubeflow/pipelines/backend/src/apiserver/client_manager" "github.com/kubeflow/pipelines/backend/src/apiserver/common" "github.com/kubeflow/pipelines/backend/src/apiserver/model" "github.com/kubeflow/pipelines/backend/src/apiserver/resource" @@ -74,9 +75,10 @@ func main() { template.Launcher = common.GetStringConfig(launcherEnv) } - clientManager := newClientManager() + clientManager := cm.NewClientManager() resourceManager := resource.NewResourceManager( &clientManager, + &resource.ResourceManagerOptions{CollectMetrics: *collectMetricsFlag}, ) err := loadSamples(resourceManager) if err != nil { @@ -135,8 +137,8 @@ func startRpcServer(resourceManager *resource.ResourceManager) { s, server.NewVisualizationServer( resourceManager, - common.GetStringConfig(visualizationServiceHost), - common.GetStringConfig(visualizationServicePort), + common.GetStringConfig(cm.VisualizationServiceHost), + common.GetStringConfig(cm.VisualizationServicePort), )) apiv1beta1.RegisterAuthServiceServer(s, server.NewAuthServer(resourceManager)) diff --git a/backend/src/apiserver/resource/resource_manager.go b/backend/src/apiserver/resource/resource_manager.go index 8f71235626..91ff10cbe3 100644 --- a/backend/src/apiserver/resource/resource_manager.go +++ b/backend/src/apiserver/resource/resource_manager.go @@ -49,11 +49,31 @@ import ( // Metric variables. Please prefix the metric names with resource_manager_. var ( + extraLabels = []string{ + // display in which Kubeflow namespace the runs were triggered + "profile", + + // display workflow name + "workflow", + } + // Count the removed workflows due to garbage collection. workflowGCCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "resource_manager_workflow_gc", Help: "The number of gabarage-collected workflows", }) + + // Count the successfull workflow runs + workflowSuccessCounter = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "resource_manager_workflow_runs_success", + Help: "The current number of successfully workflows runs", + }, extraLabels) + + // Count the failed workflow runs + workflowFailedCounter = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "resource_manager_workflow_runs_failed", + Help: "The current number of failed workflows runs", + }, extraLabels) ) type ClientManagerInterface interface { @@ -77,6 +97,10 @@ type ClientManagerInterface interface { Authenticators() []kfpauth.Authenticator } +type ResourceManagerOptions struct { + CollectMetrics bool `json:"collect_metrics,omitempty"` +} + type ResourceManager struct { experimentStore storage.ExperimentStoreInterface pipelineStore storage.PipelineStoreInterface @@ -96,9 +120,10 @@ type ResourceManager struct { time util.TimeInterface uuid util.UUIDGeneratorInterface authenticators []kfpauth.Authenticator + options *ResourceManagerOptions } -func NewResourceManager(clientManager ClientManagerInterface) *ResourceManager { +func NewResourceManager(clientManager ClientManagerInterface, options *ResourceManagerOptions) *ResourceManager { return &ResourceManager{ experimentStore: clientManager.ExperimentStore(), pipelineStore: clientManager.PipelineStore(), @@ -118,6 +143,7 @@ func NewResourceManager(clientManager ClientManagerInterface) *ResourceManager { time: clientManager.Time(), uuid: clientManager.UUID(), authenticators: clientManager.Authenticators(), + options: options, } } @@ -628,6 +654,18 @@ func (r *ResourceManager) DeleteRun(ctx context.Context, runId string) error { if err != nil { return util.Wrapf(err, "Failed to delete a run %v", runId) } + + if r.options.CollectMetrics { + if run.Conditions == string(exec.ExecutionSucceeded) { + if util.GetMetricValue(workflowSuccessCounter) > 0 { + workflowSuccessCounter.WithLabelValues(run.Namespace, run.DisplayName).Dec() + } + } else { + if util.GetMetricValue(workflowFailedCounter) > 0 { + workflowFailedCounter.WithLabelValues(run.Namespace, run.DisplayName).Dec() + } + } + } return nil } @@ -1090,8 +1128,9 @@ func (r *ResourceManager) ReportWorkflowResource(ctx context.Context, execSpec u return nil, util.NewInternalServerError(err, "Failed to delete the completed workflow for run %s", runId) } } - // TODO(jingzhang36): find a proper way to pass collectMetricsFlag here. - workflowGCCounter.Inc() + if r.options.CollectMetrics { + workflowGCCounter.Inc() + } } // If the run was Running and got terminated (activeDeadlineSeconds set to 0), // ignore its condition and mark it as such @@ -1130,8 +1169,10 @@ func (r *ResourceManager) ReportWorkflowResource(ctx context.Context, execSpec u } return nil, util.NewInternalServerError(err, "Failed to delete the obsolete workflow for run %s", runId) } - // TODO(jingzhang36): find a proper way to pass collectMetricsFlag here. - workflowGCCounter.Inc() + + if r.options.CollectMetrics { + workflowGCCounter.Inc() + } // Note, persistence agent will not retry reporting this workflow again, because updateError is a not found error. return nil, util.Wrapf(updateError, "Failed to report workflow name=%q namespace=%q runId=%q", execSpec.ExecutionName(), execSpec.ExecutionNamespace(), runId) } @@ -1219,6 +1260,20 @@ func (r *ResourceManager) ReportWorkflowResource(ctx context.Context, execSpec u return nil, util.Wrapf(err, message) } } + + if r.options.CollectMetrics { + execNamespace := execSpec.ExecutionNamespace() + execName := execSpec.ExecutionName() + + if execStatus.Condition() == exec.ExecutionSucceeded { + workflowSuccessCounter.WithLabelValues(execNamespace, execName).Inc() + } else { + glog.Errorf("pipeline '%s' finished with an error", execName) + + // also collects counts regarding retries + workflowFailedCounter.WithLabelValues(execNamespace, execName).Inc() + } + } } execSpec.SetLabels("pipeline/runid", runId) return execSpec, nil diff --git a/backend/src/apiserver/resource/resource_manager_test.go b/backend/src/apiserver/resource/resource_manager_test.go index f63fe7599e..c1221ec6ca 100644 --- a/backend/src/apiserver/resource/resource_manager_test.go +++ b/backend/src/apiserver/resource/resource_manager_test.go @@ -135,7 +135,7 @@ var testWorkflow = util.NewWorkflow(&v1alpha1.Workflow{ func initWithPipeline(t *testing.T) (*FakeClientManager, *ResourceManager, *model.Pipeline, *model.PipelineVersion) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) p1 := createPipeline("p1", "", "ns1") p, _ := manager.CreatePipeline(p1) pv1 := createPipelineVersion( @@ -155,7 +155,7 @@ func initWithPipeline(t *testing.T) (*FakeClientManager, *ResourceManager, *mode func initWithExperiment(t *testing.T) (*FakeClientManager, *ResourceManager, *model.Experiment) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) apiExperiment := &model.Experiment{Name: "e1", Namespace: "ns1"} experiment, err := manager.CreateExperiment(apiExperiment) assert.Nil(t, err) @@ -165,7 +165,7 @@ func initWithExperiment(t *testing.T) (*FakeClientManager, *ResourceManager, *mo func initWithExperimentAndPipeline(t *testing.T) (*FakeClientManager, *ResourceManager, *model.Experiment, *model.Pipeline, *model.PipelineVersion) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) apiExperiment := &model.Experiment{Name: "e1"} experiment, err := manager.CreateExperiment(apiExperiment) assert.Nil(t, err) @@ -434,7 +434,7 @@ func TestCreatePipeline(t *testing.T) { // setup store := NewFakeClientManagerOrFatalV2() defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) if test.badObjectStore { manager.objectStore = &FakeBadObjectStore{} } @@ -584,7 +584,7 @@ func TestCreatePipelineVersion(t *testing.T) { t.Run(test.msg, func(t *testing.T) { store := NewFakeClientManagerOrFatalV2() defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) // Create a pipeline before versions. p0 := createPipelineV1("my_pipeline") @@ -674,7 +674,7 @@ func TestCreatePipelineOrVersion_V2PipelineName(t *testing.T) { t.Run(fmt.Sprintf("%+v", testClone), func(t *testing.T) { store := NewFakeClientManagerOrFatalV2() defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) if test.template == "" { test.template = strings.TrimSpace(v2compatPipeline) @@ -807,7 +807,7 @@ func TestResourceManager_CreatePipelineAndPipelineVersion(t *testing.T) { t.Run(tt.name, func(t *testing.T) { store := NewFakeClientManagerOrFatalV2() defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) pipelineStore, ok := manager.pipelineStore.(*storage.PipelineStore) assert.True(t, ok) pipelineStore.SetUUIDGenerator(util.NewFakeUUIDGeneratorOrFatal(FakeUUIDOne, nil)) @@ -1001,7 +1001,7 @@ func TestGetPipelineTemplate_FromPipelineURI(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) p, _ := manager.CreatePipeline(createPipelineV1("new_pipeline")) manager.objectStore.AddFile([]byte(testWorkflow.ToStringForStore()), p.UUID) @@ -1023,7 +1023,7 @@ func TestGetPipelineTemplate_FromPipelineVersionId(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) p, _ := manager.CreatePipeline(createPipelineV1("new_pipeline")) pv := &model.PipelineVersion{ @@ -1051,7 +1051,7 @@ func TestGetPipelineTemplate_FromPipelineId(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) p, _ := manager.CreatePipeline(createPipelineV1("new_pipeline")) pv := &model.PipelineVersion{ @@ -1079,7 +1079,7 @@ func TestGetPipelineTemplate_PipelineMetadataNotFound(t *testing.T) { defer store.Close() template := []byte("workflow: foo") store.objectStore.AddFile(template, store.objectStore.GetPipelineKey(fmt.Sprint(1))) - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) _, err := manager.GetPipelineLatestTemplate("1") assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "Pipeline 1 not found") @@ -1090,7 +1090,7 @@ func TestGetPipelineTemplate_PipelineFileNotFound(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() pipeline, _ := store.PipelineStore().CreatePipeline(createPipelineV1("pipeline1")) - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) _, err := manager.GetPipelineLatestTemplate(pipeline.UUID) assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "not found") @@ -1100,7 +1100,7 @@ func TestGetPipelineTemplate_PipelineFileNotFound(t *testing.T) { func TestListPipelines(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) // Create a pipeline. p1 := createPipelineV1( @@ -1159,7 +1159,7 @@ func TestListPipelines(t *testing.T) { func TestListPipelinesV1(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) // Create a pipeline. p1 := createPipelineV1( "pipeline1", @@ -1218,7 +1218,7 @@ func TestListPipelineVersions(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) // Create a pipeline. p1 := createPipelineV1( @@ -1301,7 +1301,7 @@ func TestUpdatePipelineStatus(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) pipelineStore, ok := store.pipelineStore.(*storage.PipelineStore) assert.True(t, ok) @@ -1371,7 +1371,7 @@ func TestUpdatePipelineVersionStatus(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) pipelineStore, ok := store.pipelineStore.(*storage.PipelineStore) assert.True(t, ok) @@ -1440,7 +1440,7 @@ func TestDeletePipelineVersion(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) // Create a pipeline. p := createPipelineV1( @@ -1501,7 +1501,7 @@ func TestDeletePipelineVersion_FileError(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) // Create a pipeline. p := createPipelineV1( @@ -1543,7 +1543,7 @@ func TestDeletePipeline(t *testing.T) { initEnvVars() store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) // Create a pipeline. p1 := createPipelineV1( @@ -1999,7 +1999,7 @@ func TestCreateRun_ThroughPipelineIdAndPipelineVersion(t *testing.T) { func TestCreateRun_EmptyPipelineSpec(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") apiRun := &model.Run{ DisplayName: "run1", @@ -2016,7 +2016,7 @@ func TestCreateRun_EmptyPipelineSpec(t *testing.T) { func TestCreateRun_InvalidWorkflowSpec(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") apiRun := &model.Run{ DisplayName: "run1", @@ -2034,7 +2034,7 @@ func TestCreateRun_InvalidWorkflowSpec(t *testing.T) { func TestCreateRun_NullWorkflowSpec(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") apiRun := &model.Run{ DisplayName: "run1", @@ -2052,7 +2052,7 @@ func TestCreateRun_NullWorkflowSpec(t *testing.T) { func TestCreateRun_OverrideParametersError(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") apiRun := &model.Run{ DisplayName: "run1", @@ -2070,7 +2070,7 @@ func TestCreateRun_OverrideParametersError(t *testing.T) { func TestCreateRun_CreateWorkflowError(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") manager.execClient = client.NewFakeExecClientWithBadWorkflow() apiRun := &model.Run{ @@ -2089,7 +2089,7 @@ func TestCreateRun_CreateWorkflowError(t *testing.T) { func TestCreateRun_StoreRunMetadataError(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") store.DB().Close() apiRun := &model.Run{ @@ -2119,7 +2119,7 @@ func TestDeleteRun(t *testing.T) { func TestDeleteRun_RunNotExist(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) err := manager.DeleteRun(context.Background(), "1") assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "not found") @@ -2177,7 +2177,7 @@ func TestDeleteExperiment_ClearsDefaultExperiment(t *testing.T) { func TestDeleteExperiment_ExperimentNotExist(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) err := manager.DeleteExperiment("1") assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "not found") @@ -2221,7 +2221,7 @@ func TestTerminateRun(t *testing.T) { func TestTerminateRun_RunNotExist(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) err := manager.TerminateRun(context.Background(), "1") assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "not found") @@ -2257,7 +2257,7 @@ func TestRetryRun(t *testing.T) { func TestRetryRun_RunNotExist(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) err := manager.RetryRun(context.Background(), "1") assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "not found") @@ -2549,7 +2549,7 @@ func TestCreateJob_ThroughPipelineIdAndPipelineVersion(t *testing.T) { func TestCreateJob_EmptyPipelineSpec(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") job := &model.Job{ DisplayName: "pp 1", @@ -2567,7 +2567,7 @@ func TestCreateJob_EmptyPipelineSpec(t *testing.T) { func TestCreateJob_InvalidWorkflowSpec(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") job := &model.Job{ K8SName: "pp 1", @@ -2586,7 +2586,7 @@ func TestCreateJob_InvalidWorkflowSpec(t *testing.T) { func TestCreateJob_NullWorkflowSpec(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, _ := manager.CreateDefaultExperiment("") job := &model.Job{ K8SName: "pp 1", @@ -2667,7 +2667,7 @@ func TestEnableJob(t *testing.T) { func TestEnableJob_JobNotExist(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) err := manager.ChangeJobMode(context.Background(), "1", false) assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "Job 1 not found") @@ -2733,7 +2733,7 @@ func TestDeleteJob(t *testing.T) { func TestDeleteJob_JobNotExist(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) err := manager.DeleteJob(context.Background(), "1") assert.Equal(t, codes.NotFound, err.(*util.UserError).ExternalStatusCode()) assert.Contains(t, err.Error(), "Job 1 not found") @@ -2900,7 +2900,7 @@ func TestReportWorkflowResource_WorkflowMissingRunID(t *testing.T) { func TestReportWorkflowResource_RunNotFound(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) ctx := context.Background() defer store.Close() workflow := util.NewWorkflow(&v1alpha1.Workflow{ @@ -3184,7 +3184,7 @@ func TestReportScheduledWorkflowResource_Success_withRuntimeParamsV2(t *testing. func TestReportScheduledWorkflowResource_Error(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) manager.CreateDefaultExperiment("") // Create pipeline workflow := util.NewWorkflow(&v1alpha1.Workflow{ @@ -3324,7 +3324,7 @@ func TestReadArtifact_WorkflowNoStatus_NotFound(t *testing.T) { func TestReadArtifact_NoRun_NotFound(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) _, err := manager.ReadArtifact("run-1", "node-1", "artifact-1") assert.True(t, util.IsUserErrorCodeMatch(err, codes.NotFound)) @@ -3938,7 +3938,7 @@ spec: func TestCreateDefaultExperiment(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, err := manager.CreateDefaultExperiment("") assert.Nil(t, err) @@ -3962,7 +3962,7 @@ func TestCreateDefaultExperiment_MultiUser(t *testing.T) { store := NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) defer store.Close() - manager := NewResourceManager(store) + manager := NewResourceManager(store, &ResourceManagerOptions{CollectMetrics: false}) experimentID, err := manager.CreateDefaultExperiment("multi-user") assert.Nil(t, err) diff --git a/backend/src/apiserver/server/api_util_test.go b/backend/src/apiserver/server/api_util_test.go index e69a81ed71..e004e4639e 100644 --- a/backend/src/apiserver/server/api_util_test.go +++ b/backend/src/apiserver/server/api_util_test.go @@ -108,7 +108,7 @@ func TestValidateExperimentResourceReference_UnexpectedRelationship(t *testing.T func TestValidateExperimentResourceReference_ExperimentNotExist(t *testing.T) { clients := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - manager := resource.NewResourceManager(clients) + manager := resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) defer clients.Close() err := ValidateExperimentResourceReference(manager, validReference) assert.NotNil(t, err) @@ -183,7 +183,7 @@ func TestValidatePipelineSpecAndResourceReferences_InvalidPipelineVersionId(t *t func TestValidatePipelineSpecAndResourceReferences_PipelineIdNotParentOfPipelineVersionId(t *testing.T) { clients := initWithExperimentsAndTwoPipelineVersions(t) - manager := resource.NewResourceManager(clients) + manager := resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) defer clients.Close() spec := &apiv1beta1.PipelineSpec{ PipelineId: NonDefaultFakeUUID, diff --git a/backend/src/apiserver/server/experiment_server_test.go b/backend/src/apiserver/server/experiment_server_test.go index 1f6449c73e..44a1dc2d04 100644 --- a/backend/src/apiserver/server/experiment_server_test.go +++ b/backend/src/apiserver/server/experiment_server_test.go @@ -35,7 +35,7 @@ import ( func TestCreateExperimentV1(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "ex1", Description: "first experiment"} @@ -59,7 +59,7 @@ func TestCreateExperimentV1(t *testing.T) { func TestCreateExperiment(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment"} @@ -78,7 +78,7 @@ func TestCreateExperiment(t *testing.T) { func TestCreateExperimentV1_Failed(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "ex1", Description: "first experiment"} clientManager.DB().Close() @@ -89,7 +89,7 @@ func TestCreateExperimentV1_Failed(t *testing.T) { func TestCreateExperiment_Failed(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment"} clientManager.DB().Close() @@ -100,7 +100,7 @@ func TestCreateExperiment_Failed(t *testing.T) { func TestCreateExperiment_EmptyName(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "", Description: "first experiment"} clientManager.DB().Close() @@ -111,7 +111,7 @@ func TestCreateExperiment_EmptyName(t *testing.T) { func TestCreateExperimentV1_EmptyName(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "", Description: "first experiment"} clientManager.DB().Close() @@ -186,7 +186,7 @@ func TestCreateExperimentV1_Multiuser(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} tests := []struct { @@ -360,7 +360,7 @@ func TestCreateExperimentV1_Multiuser(t *testing.T) { } for _, tt := range tests { clientManager.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(tt.fakeId, nil)) - resourceManager = resource.NewResourceManager(clientManager) + resourceManager = resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server = ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} got, err := server.CreateExperimentV1(ctx, &apiv1beta1.CreateExperimentRequest{Experiment: tt.experiment}) if tt.wantError { @@ -382,7 +382,7 @@ func TestCreateExperiment_Multiuser(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} tests := []struct { @@ -446,7 +446,7 @@ func TestCreateExperiment_Multiuser(t *testing.T) { func TestGetExperimentV1(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "ex1", Description: "first experiment"} @@ -472,7 +472,7 @@ func TestGetExperimentV1(t *testing.T) { func TestGetExperiment(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment"} @@ -493,7 +493,7 @@ func TestGetExperiment(t *testing.T) { func TestGetExperimentV1_Failed(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "ex1", Description: "first experiment"} @@ -507,7 +507,7 @@ func TestGetExperimentV1_Failed(t *testing.T) { func TestGetExperiment_Failed(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment"} @@ -570,7 +570,7 @@ func TestGetExperimentV1_Multiuser(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} resourceReferences := []*apiv1beta1.ResourceReference{ { @@ -606,7 +606,7 @@ func TestGetExperiment_Multiuser(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{ DisplayName: "exp1", @@ -631,7 +631,7 @@ func TestGetExperiment_Multiuser(t *testing.T) { func TestListExperimentsV1(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "ex1", Description: "first experiment"} @@ -659,7 +659,7 @@ func TestListExperimentsV1(t *testing.T) { func TestListExperiments(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment"} @@ -680,7 +680,7 @@ func TestListExperiments(t *testing.T) { func TestListExperimentsV1_Failed(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "ex1", Description: "first experiment"} @@ -694,7 +694,7 @@ func TestListExperimentsV1_Failed(t *testing.T) { func TestListExperiments_Failed(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment"} @@ -762,7 +762,7 @@ func TestListExperimentsV1_Multiuser(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} resourceReferences := []*apiv1beta1.ResourceReference{ @@ -889,7 +889,7 @@ func TestListExperiments_Multiuser_NoDefault(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{ DisplayName: "exp1", @@ -967,7 +967,7 @@ func TestArchiveAndUnarchiveExperimentV1(t *testing.T) { _, err := runServer.CreateRunV1(nil, &apiv1beta1.CreateRunRequest{Run: run1}) assert.Nil(t, err) clients.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(FakeUUIDOne, nil)) - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) runServer = NewRunServer(manager, &RunServerOptions{CollectMetrics: false}) run2 := &apiv1beta1.Run{ Name: "run2", @@ -976,7 +976,7 @@ func TestArchiveAndUnarchiveExperimentV1(t *testing.T) { _, err = runServer.CreateRunV1(nil, &apiv1beta1.CreateRunRequest{Run: run2}) assert.Nil(t, err) clients.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(DefaultFakeUUID, nil)) - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) jobServer := NewJobServer(manager, &JobServerOptions{CollectMetrics: false}) job1 := &apiv1beta1.Job{ Name: "name1", @@ -1043,7 +1043,7 @@ func TestArchiveAndUnarchiveExperiment(t *testing.T) { _, err := runServer.CreateRunV1(nil, &apiv1beta1.CreateRunRequest{Run: run1}) assert.Nil(t, err) clients.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(FakeUUIDOne, nil)) - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) runServer = NewRunServer(manager, &RunServerOptions{CollectMetrics: false}) run2 := &apiv1beta1.Run{ Name: "run2", @@ -1052,7 +1052,7 @@ func TestArchiveAndUnarchiveExperiment(t *testing.T) { _, err = runServer.CreateRunV1(nil, &apiv1beta1.CreateRunRequest{Run: run2}) assert.Nil(t, err) clients.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(DefaultFakeUUID, nil)) - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) jobServer := NewJobServer(manager, &JobServerOptions{CollectMetrics: false}) job1 := &apiv1beta1.Job{ Name: "name1", @@ -1107,7 +1107,7 @@ func TestArchiveAndUnarchiveExperiment(t *testing.T) { // deleting an experiment that does not exist in single user mode, for V2 api. func TestDeleteExperiments_SingleUser(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment"} resultExperiment, err := server.CreateExperiment(nil, &apiV2beta1.CreateExperimentRequest{Experiment: experiment}) @@ -1125,7 +1125,7 @@ func TestDeleteExperiments_SingleUser(t *testing.T) { // deleting an experiment that does not exist in single user mode, for V1 api. func TestDeleteExperimentsV1_SingleUser(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiv1beta1.Experiment{Name: "ex1", Description: "first experiment"} resultExperiment, err := server.CreateExperimentV1(nil, &apiv1beta1.CreateExperimentRequest{Experiment: experiment}) @@ -1148,7 +1148,7 @@ func TestDeleteExperiments_MultiUser(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} experiment := &apiV2beta1.Experiment{DisplayName: "ex1", Description: "first experiment", Namespace: "ns1"} resultExperiment, err := server.CreateExperiment(ctx, &apiV2beta1.CreateExperimentRequest{Experiment: experiment}) @@ -1171,7 +1171,7 @@ func TestDeleteExperimentsV1_MultiUser(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := ExperimentServer{resourceManager: resourceManager, options: &ExperimentServerOptions{CollectMetrics: false}} resourceReferences := []*apiv1beta1.ResourceReference{ { diff --git a/backend/src/apiserver/server/fakes_test.go b/backend/src/apiserver/server/fakes_test.go index 0027fd6a40..70e50000bc 100644 --- a/backend/src/apiserver/server/fakes_test.go +++ b/backend/src/apiserver/server/fakes_test.go @@ -144,7 +144,7 @@ func initEnvVars() { func initWithExperiment(t *testing.T) (*resource.FakeClientManager, *resource.ResourceManager, *model.Experiment) { initEnvVars() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) var apiExperiment *apiv1beta1.Experiment if common.IsMultiUserMode() { apiExperiment = &apiv1beta1.Experiment{ @@ -178,7 +178,7 @@ func initWithExperiment_SubjectAccessReview_Unauthorized(t *testing.T) (*resourc initEnvVars() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) clientManager.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) apiExperiment := &apiv1beta1.Experiment{Name: "exp1"} if common.IsMultiUserMode() { apiExperiment = &apiv1beta1.Experiment{ @@ -202,7 +202,7 @@ func initWithExperiment_SubjectAccessReview_Unauthorized(t *testing.T) (*resourc func initWithExperimentAndPipelineVersion(t *testing.T) (*resource.FakeClientManager, *resource.ResourceManager, *model.Experiment, *model.PipelineVersion) { initEnvVars() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) // Create an experiment. apiExperiment := &apiv1beta1.Experiment{Name: "exp1"} @@ -241,7 +241,7 @@ func initWithExperimentAndPipelineVersion(t *testing.T) (*resource.FakeClientMan func initWithExperimentsAndTwoPipelineVersions(t *testing.T) *resource.FakeClientManager { initEnvVars() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) // Create an experiment. apiExperiment := &apiv1beta1.Experiment{Name: "exp1"} @@ -268,7 +268,7 @@ func initWithExperimentsAndTwoPipelineVersions(t *testing.T) *resource.FakeClien ) assert.Nil(t, err) clientManager.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal("123e4567-e89b-12d3-a456-426655441001", nil)) - resourceManager = resource.NewResourceManager(clientManager) + resourceManager = resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) _, err = resourceManager.CreatePipelineVersion( &model.PipelineVersion{ Name: "pipeline_version", @@ -277,7 +277,7 @@ func initWithExperimentsAndTwoPipelineVersions(t *testing.T) *resource.FakeClien ) assert.Nil(t, err) clientManager.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(NonDefaultFakeUUID, nil)) - resourceManager = resource.NewResourceManager(clientManager) + resourceManager = resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) // Create another pipeline and then pipeline version. p1, err := resourceManager.CreatePipeline( &model.Pipeline{ @@ -299,7 +299,7 @@ func initWithExperimentsAndTwoPipelineVersions(t *testing.T) *resource.FakeClien assert.Nil(t, err) clientManager.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal("123e4567-e89b-12d3-a456-426655441002", nil)) - resourceManager = resource.NewResourceManager(clientManager) + resourceManager = resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) _, err = resourceManager.CreatePipelineVersion( &model.PipelineVersion{ Name: "another_pipeline_version", diff --git a/backend/src/apiserver/server/job_server_test.go b/backend/src/apiserver/server/job_server_test.go index 2b29e1c7ac..6e53a2edb5 100644 --- a/backend/src/apiserver/server/job_server_test.go +++ b/backend/src/apiserver/server/job_server_test.go @@ -317,7 +317,7 @@ func TestCreateJob_NoResRefs(t *testing.T) { clients, manager, _, _ := initWithExperimentAndPipelineVersion(t) defer clients.Close() clients.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(DefaultFakeIdTwo, nil)) - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) server := NewJobServer(manager, &JobServerOptions{CollectMetrics: false}) apiJob := &apiv1beta1.Job{ Name: "job1", @@ -618,7 +618,7 @@ func TestGetJob_Unauthorized(t *testing.T) { assert.Nil(t, err) clients.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) server = NewJobServer(manager, &JobServerOptions{CollectMetrics: false}) _, err = server.GetJob(ctx, &apiv1beta1.GetJobRequest{Id: job.Id}) @@ -825,7 +825,7 @@ func TestEnableJob_Unauthorized(t *testing.T) { assert.Nil(t, err) clients.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) server = NewJobServer(manager, &JobServerOptions{CollectMetrics: false}) _, err = server.EnableJob(ctx, &apiv1beta1.EnableJobRequest{Id: job.Id}) @@ -870,7 +870,7 @@ func TestDisableJob_Unauthorized(t *testing.T) { assert.Nil(t, err) clients.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() - manager = resource.NewResourceManager(clients) + manager = resource.NewResourceManager(clients, &resource.ResourceManagerOptions{CollectMetrics: false}) server = NewJobServer(manager, &JobServerOptions{CollectMetrics: false}) _, err = server.DisableJob(ctx, &apiv1beta1.DisableJobRequest{Id: job.Id}) diff --git a/backend/src/apiserver/server/pipeline_server_test.go b/backend/src/apiserver/server/pipeline_server_test.go index 784385e46b..8ff7831fe5 100644 --- a/backend/src/apiserver/server/pipeline_server_test.go +++ b/backend/src/apiserver/server/pipeline_server_test.go @@ -56,7 +56,7 @@ func TestCreatePipelineV1_YAML(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipeline, err := pipelineServer.CreatePipelineV1(context.Background(), &api.CreatePipelineRequest{ @@ -85,7 +85,7 @@ func TestCreatePipelineV1_LargeFile(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipeline, err := pipelineServer.CreatePipelineV1(context.Background(), &api.CreatePipelineRequest{ @@ -114,7 +114,7 @@ func TestCreatePipelineV1_Tarball(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipeline, err := pipelineServer.CreatePipelineV1(context.Background(), &api.CreatePipelineRequest{ @@ -144,7 +144,7 @@ func TestCreatePipelineV1_InvalidYAML(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} createdPipeline, err := pipelineServer.CreatePipelineV1( @@ -166,7 +166,7 @@ func TestCreatePipelineV1_InvalidURL(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} createdPipeline, err := pipelineServer.CreatePipelineV1( @@ -188,7 +188,7 @@ func TestCreatePipelineV1_MissingUrl(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} createdPipeline, err := pipelineServer.CreatePipelineV1( @@ -218,7 +218,7 @@ func TestCreatePipelineV1_ExistingPipeline(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipelineServer.CreatePipelineV1( @@ -267,7 +267,7 @@ func TestCreatePipelineVersionV1_YAML(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal( util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{ resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}, @@ -312,7 +312,7 @@ func TestCreatePipelineVersion_InvalidYAML(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} _, err := pipelineServer.CreatePipelineVersionV1( @@ -344,7 +344,7 @@ func TestCreatePipelineVersion_Tarball(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipelineVersion, err := pipelineServer.CreatePipelineVersionV1( @@ -386,7 +386,7 @@ func TestCreatePipelineVersion_InvalidURL(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} _, err := pipelineServer.CreatePipelineVersionV1(context.Background(), &api.CreatePipelineVersionRequest{ @@ -417,7 +417,7 @@ func TestListPipelineVersion_NoResourceKey(t *testing.T) { defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} @@ -433,7 +433,7 @@ func TestListPipelinesPublic(t *testing.T) { // Close the server when test finishes defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} _, err := pipelineServer.ListPipelinesV1(context.Background(), @@ -452,7 +452,7 @@ func TestGetPipelineByName_OK(t *testing.T) { // Close the server when test finishes defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipeline, err := pipelineServer.CreatePipelineV1(context.Background(), &api.CreatePipelineRequest{ Pipeline: &api.Pipeline{ @@ -485,7 +485,7 @@ func TestGetPipelineByName_Shared_OK(t *testing.T) { // Close the server when test finishes defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipeline, err := pipelineServer.CreatePipelineV1(context.Background(), &api.CreatePipelineRequest{ Pipeline: &api.Pipeline{ @@ -513,7 +513,7 @@ func TestGetPipelineByName_NotFound(t *testing.T) { // Close the server when test finishes defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} _, err := pipelineServer.GetPipelineByNameV1(context.Background(), &api.GetPipelineByNameRequest{ @@ -527,7 +527,7 @@ func TestGetPipelineByName_WrongNameSpace(t *testing.T) { // Close the server when test finishes defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipeline, err := pipelineServer.CreatePipelineV1(context.Background(), &api.CreatePipelineRequest{ Pipeline: &api.Pipeline{ @@ -563,7 +563,7 @@ func TestCreatePipelineVersionAndCheckLatestVersion(t *testing.T) { // Close the server when test finishes defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipeline, err := pipelineServer.CreatePipelineV1(context.Background(), &api.CreatePipelineRequest{ @@ -579,7 +579,7 @@ func TestCreatePipelineVersionAndCheckLatestVersion(t *testing.T) { assert.NotNil(t, pipeline.DefaultVersion.Id) clientManager.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal("123e4567-e89b-12d3-a456-526655440001", nil)) - resourceManager = resource.NewResourceManager(clientManager) + resourceManager = resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer = PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} pipelineVersion, err := pipelineServer.CreatePipelineVersionV1( @@ -634,7 +634,7 @@ func TestPipelineServer_CreatePipeline(t *testing.T) { httpServer := getMockServer(t) defer httpServer.Close() clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} type args struct { @@ -699,7 +699,7 @@ func TestPipelineServer_CreatePipeline(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { clientManager.UpdateUUID(util.NewFakeUUIDGeneratorOrFatal(tt.id, nil)) - resourceManager = resource.NewResourceManager(clientManager) + resourceManager = resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer = PipelineServer{resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}} got, err := pipelineServer.CreatePipeline(context.Background(), &apiv2.CreatePipelineRequest{Pipeline: tt.arg}) if tt.wantErr { @@ -850,7 +850,7 @@ func TestPipelineServer_CreatePipelineAndVersion_v2(t *testing.T) { for _, tt := range tests { clientManager := resource.NewFakeClientManagerOrFatal( util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) pipelineServer := PipelineServer{ resourceManager: resourceManager, httpClient: httpServer.Client(), options: &PipelineServerOptions{CollectMetrics: false}, } diff --git a/backend/src/apiserver/server/pipeline_upload_server_test.go b/backend/src/apiserver/server/pipeline_upload_server_test.go index 931f7b5f0f..e26eed60b9 100644 --- a/backend/src/apiserver/server/pipeline_upload_server_test.go +++ b/backend/src/apiserver/server/pipeline_upload_server_test.go @@ -614,14 +614,14 @@ func setupWriter(text string) (*bytes.Buffer, *multipart.Writer) { func setupClientManagerAndServer() (*resource.FakeClientManager, PipelineUploadServer) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := PipelineUploadServer{resourceManager: resourceManager, options: &PipelineUploadServerOptions{CollectMetrics: false}} return clientManager, server } func updateClientManager(clientManager *resource.FakeClientManager, uuid util.UUIDGeneratorInterface) PipelineUploadServer { clientManager.UpdateUUID(uuid) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) server := PipelineUploadServer{resourceManager: resourceManager, options: &PipelineUploadServerOptions{CollectMetrics: false}} return server } diff --git a/backend/src/apiserver/server/run_server.go b/backend/src/apiserver/server/run_server.go index 1a6cef80bf..a850fa9bdc 100644 --- a/backend/src/apiserver/server/run_server.go +++ b/backend/src/apiserver/server/run_server.go @@ -85,8 +85,6 @@ var ( Help: "The total number of RetryRun requests", }) - // TODO(jingzhang36): error count and success count. - runCount = promauto.NewGauge(prometheus.GaugeOpts{ Name: "run_server_run_count", Help: "The current number of runs in Kubeflow Pipelines instance", @@ -332,7 +330,9 @@ func (s *RunServer) DeleteRunV1(ctx context.Context, request *apiv1beta1.DeleteR return nil, util.Wrap(err, "Failed to delete a v1beta1 run") } if s.options.CollectMetrics { - runCount.Dec() + if util.GetMetricValue(runCount) > 0 { + runCount.Dec() + } } return &empty.Empty{}, nil } diff --git a/backend/src/apiserver/server/run_server_test.go b/backend/src/apiserver/server/run_server_test.go index 92a544763b..89aa763f01 100644 --- a/backend/src/apiserver/server/run_server_test.go +++ b/backend/src/apiserver/server/run_server_test.go @@ -1262,7 +1262,7 @@ func TestReportRunMetricsV1_Unauthorized(t *testing.T) { clientManager, resourceManager, runDetails := initWithOneTimeRun(t) defer clientManager.Close() clientManager.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() - resourceManager = resource.NewResourceManager(clientManager) + resourceManager = resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) runServer := RunServer{resourceManager: resourceManager, options: &RunServerOptions{CollectMetrics: false}} _, err := runServer.ReportRunMetricsV1(ctx, &apiv1beta1.ReportRunMetricsRequest{ @@ -1502,7 +1502,7 @@ func TestReadArtifactsV1_Unauthorized(t *testing.T) { // make the following request unauthorized clientManager.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) runServer := RunServer{resourceManager: resourceManager, options: &RunServerOptions{CollectMetrics: false}} artifact := &apiv1beta1.ReadArtifactRequest{ @@ -1521,7 +1521,7 @@ func TestReadArtifactsV1_Unauthorized(t *testing.T) { func TestReadArtifactsV1_Run_NotFound(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - manager := resource.NewResourceManager(clientManager) + manager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) runServer := RunServer{resourceManager: manager, options: &RunServerOptions{CollectMetrics: false}} artifact := &apiv1beta1.ReadArtifactRequest{ RunId: "Wrong_RUN_UUID", diff --git a/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml b/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml index b9ba567619..e7d2be1cc3 100644 --- a/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml +++ b/backend/src/apiserver/server/test/xgboost_sample_pipeline.yaml @@ -237,7 +237,7 @@ components: artifactType: schemaTitle: system.Artifact schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root +defaultPipelineRoot: minio://dummy_root deploymentSpec: executors: exec-chicago-taxi-trips-dataset: diff --git a/backend/src/apiserver/server/visualization_server_test.go b/backend/src/apiserver/server/visualization_server_test.go index 0f32ec7e54..da6ecfee1f 100644 --- a/backend/src/apiserver/server/visualization_server_test.go +++ b/backend/src/apiserver/server/visualization_server_test.go @@ -276,7 +276,7 @@ func TestCreateVisualization_Unauthorized(t *testing.T) { clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) clientManager.SubjectAccessReviewClientFake = client.NewFakeSubjectAccessReviewClientUnauthorized() - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) defer clientManager.Close() server := &VisualizationServer{ @@ -316,7 +316,7 @@ func TestCreateVisualization_Unauthenticated(t *testing.T) { ctx := metadata.NewIncomingContext(context.Background(), md) clientManager := resource.NewFakeClientManagerOrFatal(util.NewFakeTimeForEpoch()) - resourceManager := resource.NewResourceManager(clientManager) + resourceManager := resource.NewResourceManager(clientManager, &resource.ResourceManagerOptions{CollectMetrics: false}) defer clientManager.Close() server := &VisualizationServer{ diff --git a/backend/src/apiserver/visualization/requirements-test.txt b/backend/src/apiserver/visualization/requirements-test.txt index d9d5fbfd16..dc9791f7f2 100644 --- a/backend/src/apiserver/visualization/requirements-test.txt +++ b/backend/src/apiserver/visualization/requirements-test.txt @@ -1 +1 @@ -snapshottest==0.5.1 +snapshottest==0.6.0 diff --git a/backend/src/apiserver/visualization/requirements.in b/backend/src/apiserver/visualization/requirements.in index a5b6919c07..61ebd737ba 100644 --- a/backend/src/apiserver/visualization/requirements.in +++ b/backend/src/apiserver/visualization/requirements.in @@ -4,14 +4,16 @@ google-api-python-client==1.7.* itables==0.1.0 ipykernel==5.1.1 ipython==7.12.0 +jinja2==2.11.3 jupyter_client==5.3.* +markupsafe==2.0.1 nbconvert==5.5.0 nbformat==4.4.0 -scikit_learn==0.21.2 -tensorflow==2.5.1 -tensorflow-metadata==1.2.* -tensorflow-model-analysis==0.33.* -tensorflow-data-validation==1.2.* -tensorflow-serving-api==2.5.1 +scikit-learn==0.24.2 +tensorflow==2.10.1 +tensorflow-metadata==1.9.* +tensorflow-model-analysis==0.40.* +tensorflow-data-validation==1.9.* +tensorflow-serving-api==2.10.1 tornado==6.* mistune<2.0.0 \ No newline at end of file diff --git a/backend/src/apiserver/visualization/requirements.txt b/backend/src/apiserver/visualization/requirements.txt index 1e0e6dd825..bd95a01914 100644 --- a/backend/src/apiserver/visualization/requirements.txt +++ b/backend/src/apiserver/visualization/requirements.txt @@ -1,153 +1,570 @@ # -# This file is autogenerated by pip-compile -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # -# pip-compile --output-file=- - +# pip-compile --output-file=- --resolver=backtracking - # -absl-py==0.12.0 # via tensorboard, tensorflow, tensorflow-data-validation, tensorflow-metadata, tensorflow-model-analysis, tfx-bsl -apache-beam[gcp]==2.34.0 # via tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl -argon2-cffi-bindings==21.2.0 # via argon2-cffi -argon2-cffi==21.3.0 # via notebook -astunparse==1.6.3 # via tensorflow -attrs==21.2.0 # via jsonschema -avro-python3==1.9.2.1 # via apache-beam -backcall==0.2.0 # via ipython -bleach==4.1.0 # via nbconvert -bokeh==1.2.0 # via -r - -cached-property==1.5.2 # via h5py -cachetools==4.2.4 # via apache-beam, google-auth -certifi==2021.10.8 # via requests -cffi==1.15.0 # via argon2-cffi-bindings -charset-normalizer==2.0.9 # via requests -crcmod==1.7 # via apache-beam -dataclasses==0.8 # via apache-beam, argon2-cffi, libcst, werkzeug -decorator==5.1.0 # via gcsfs, ipython, traitlets -defusedxml==0.7.1 # via nbconvert -dill==0.3.1.1 # via apache-beam -docopt==0.6.2 # via hdfs -entrypoints==0.3 # via nbconvert -fastavro==1.4.7 # via apache-beam -fasteners==0.16.3 # via google-apitools -flatbuffers==1.12 # via tensorflow -future==0.18.2 # via apache-beam -gast==0.4.0 # via tensorflow -gcsfs==0.2.3 # via -r - -google-api-core[grpc,grpcgcp]==1.31.5 # via google-cloud-bigquery, google-cloud-bigquery-storage, google-cloud-bigtable, google-cloud-core, google-cloud-datastore, google-cloud-dlp, google-cloud-language, google-cloud-pubsub, google-cloud-recommendations-ai, google-cloud-spanner, google-cloud-videointelligence, google-cloud-vision -google-api-python-client==1.7.12 # via -r -, tfx-bsl -google-apitools==0.5.31 # via apache-beam -google-auth-httplib2==0.1.0 # via google-api-python-client -google-auth-oauthlib==0.4.6 # via gcsfs, tensorboard -google-auth==1.35.0 # via apache-beam, gcsfs, google-api-core, google-api-python-client, google-auth-httplib2, google-auth-oauthlib, google-cloud-core, tensorboard -google-cloud-bigquery-storage==2.10.1 # via apache-beam -google-cloud-bigquery==2.20.0 # via apache-beam, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl -google-cloud-bigtable==1.7.0 # via apache-beam -google-cloud-core==1.7.2 # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore, google-cloud-spanner -google-cloud-datastore==1.15.3 # via apache-beam -google-cloud-dlp==1.0.0 # via apache-beam -google-cloud-language==1.3.0 # via apache-beam -google-cloud-pubsub==1.7.0 # via apache-beam -google-cloud-recommendations-ai==0.2.0 # via apache-beam -google-cloud-spanner==1.19.1 # via apache-beam -google-cloud-videointelligence==1.16.1 # via apache-beam -google-cloud-vision==1.0.0 # via apache-beam -google-crc32c==1.3.0 # via google-resumable-media -google-pasta==0.2.0 # via tensorflow -google-resumable-media==1.3.3 # via google-cloud-bigquery -googleapis-common-protos[grpc]==1.54.0 # via google-api-core, grpc-google-iam-v1, tensorflow-metadata -grpc-google-iam-v1==0.12.3 # via google-cloud-bigtable, google-cloud-pubsub, google-cloud-spanner -grpcio-gcp==0.2.2 # via apache-beam, google-api-core -grpcio==1.34.1 # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, grpcio-gcp, tensorboard, tensorflow, tensorflow-serving-api -h5py==3.1.0 # via tensorflow -hdfs==2.6.0 # via apache-beam -httplib2==0.19.1 # via apache-beam, google-api-python-client, google-apitools, google-auth-httplib2, oauth2client -idna==3.3 # via requests -importlib-metadata==4.8.3 # via jsonschema, markdown -ipykernel==5.1.1 # via -r -, ipywidgets, notebook -ipython-genutils==0.2.0 # via ipywidgets, nbformat, notebook, traitlets -ipython==7.12.0 # via -r -, ipykernel, ipywidgets, tensorflow-model-analysis -ipywidgets==7.6.5 # via tensorflow-model-analysis -itables==0.1.0 # via -r - -jedi==0.18.1 # via ipython -jinja2==3.0.3 # via bokeh, nbconvert, notebook -joblib==0.14.1 # via scikit-learn, tensorflow-data-validation -jsonschema==3.2.0 # via nbformat -jupyter-client==5.3.5 # via -r -, ipykernel, notebook -jupyter-core==4.9.1 # via jupyter-client, nbconvert, nbformat, notebook -jupyterlab-widgets==1.0.2 # via ipywidgets -keras-nightly==2.5.0.dev2021032900 # via tensorflow -keras-preprocessing==1.1.2 # via tensorflow -libcst==0.3.23 # via google-cloud-bigquery-storage -markdown==3.3.6 # via tensorboard -markupsafe==2.0.1 # via jinja2 -mistune==0.8.4 # via -r -, nbconvert -mypy-extensions==0.4.3 # via typing-inspect -nbconvert==5.5.0 # via -r -, notebook -nbformat==4.4.0 # via -r -, ipywidgets, nbconvert, notebook -nest-asyncio==1.5.4 # via notebook -notebook==6.4.6 # via widgetsnbextension -numpy==1.19.5 # via apache-beam, bokeh, h5py, keras-preprocessing, opt-einsum, pandas, pyarrow, scikit-learn, scipy, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl -oauth2client==4.1.3 # via apache-beam, google-apitools -oauthlib==3.1.1 # via requests-oauthlib -opt-einsum==3.3.0 # via tensorflow -orjson==3.6.1 # via apache-beam -packaging==21.3 # via bleach, bokeh, google-api-core, google-cloud-bigquery -pandas==1.1.5 # via itables, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl -pandocfilters==1.5.0 # via nbconvert -parso==0.8.3 # via jedi -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -pillow==8.4.0 # via bokeh -prometheus-client==0.12.0 # via notebook -prompt-toolkit==3.0.24 # via ipython -proto-plus==1.19.8 # via google-cloud-bigquery, google-cloud-bigquery-storage, google-cloud-recommendations-ai -protobuf==3.19.1 # via apache-beam, google-api-core, google-cloud-bigquery, googleapis-common-protos, proto-plus, tensorboard, tensorflow, tensorflow-data-validation, tensorflow-metadata, tensorflow-model-analysis, tensorflow-serving-api, tfx-bsl -ptyprocess==0.7.0 # via pexpect, terminado -pyarrow==2.0.0 # via apache-beam, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl -pyasn1-modules==0.2.8 # via google-auth, oauth2client -pyasn1==0.4.8 # via oauth2client, pyasn1-modules, rsa -pycparser==2.21 # via cffi -pydot==1.4.2 # via apache-beam -pygments==2.10.0 # via ipython, nbconvert -pymongo==3.12.3 # via apache-beam -pyparsing==2.4.7 # via httplib2, packaging, pydot -pyrsistent==0.18.0 # via jsonschema -python-dateutil==2.8.2 # via apache-beam, bokeh, jupyter-client, pandas -pytz==2021.3 # via apache-beam, google-api-core, pandas -pyyaml==6.0 # via bokeh, libcst -pyzmq==22.3.0 # via jupyter-client, notebook -requests-oauthlib==1.3.0 # via google-auth-oauthlib -requests==2.26.0 # via apache-beam, gcsfs, google-api-core, google-cloud-bigquery, hdfs, requests-oauthlib, tensorboard -rsa==4.8 # via google-auth, oauth2client -scikit_learn==0.21.2 # via -r - -scipy==1.5.4 # via scikit-learn, tensorflow-model-analysis -send2trash==1.8.0 # via notebook -six==1.15.0 # via absl-py, astunparse, bleach, bokeh, fasteners, google-api-core, google-api-python-client, google-apitools, google-auth, google-auth-httplib2, google-cloud-core, google-pasta, google-resumable-media, grpcio, hdfs, jsonschema, keras-preprocessing, oauth2client, python-dateutil, tensorflow, tensorflow-data-validation, tensorflow-model-analysis, traitlets -tensorboard-data-server==0.6.1 # via tensorboard -tensorboard-plugin-wit==1.8.0 # via tensorboard -tensorboard==2.7.0 # via tensorflow -tensorflow-data-validation==1.2.0 # via -r - -tensorflow-estimator==2.5.0 # via tensorflow -tensorflow-metadata==1.2.0 # via -r -, tensorflow-data-validation, tensorflow-model-analysis, tfx-bsl -tensorflow-model-analysis==0.33.0 # via -r - -tensorflow-serving-api==2.5.1 # via -r -, tfx-bsl -tensorflow==2.5.1 # via -r -, tensorflow-data-validation, tensorflow-model-analysis, tensorflow-serving-api, tfx-bsl -termcolor==1.1.0 # via tensorflow -terminado==0.12.1 # via notebook -testpath==0.5.0 # via nbconvert -tfx-bsl==1.2.0 # via tensorflow-data-validation, tensorflow-model-analysis -tornado==6.1 # via -r -, bokeh, ipykernel, jupyter-client, notebook, terminado -traitlets==4.3.3 # via ipykernel, ipython, ipywidgets, jupyter-client, jupyter-core, nbconvert, nbformat, notebook -typing-extensions==3.7.4.3 # via apache-beam, argon2-cffi, importlib-metadata, libcst, tensorflow, typing-inspect -typing-inspect==0.7.1 # via libcst -uritemplate==3.0.1 # via google-api-python-client -urllib3==1.26.7 # via requests -wcwidth==0.2.5 # via prompt-toolkit -webencodings==0.5.1 # via bleach -werkzeug==2.0.2 # via tensorboard -wheel==0.37.1 # via astunparse, tensorboard, tensorflow -widgetsnbextension==3.5.2 # via ipywidgets -wrapt==1.12.1 # via tensorflow -zipp==3.6.0 # via importlib-metadata +absl-py==1.4.0 + # via + # tensorboard + # tensorflow + # tensorflow-data-validation + # tensorflow-metadata + # tensorflow-model-analysis + # tfx-bsl +apache-beam[gcp]==2.46.0 + # via + # tensorflow-data-validation + # tensorflow-model-analysis + # tfx-bsl +argon2-cffi==23.1.0 + # via + # jupyter-server + # notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +astunparse==1.6.3 + # via tensorflow +attrs==23.1.0 + # via + # jsonschema + # referencing +backcall==0.2.0 + # via ipython +bleach==6.0.0 + # via nbconvert +bokeh==1.2.0 + # via -r - +cachetools==4.2.4 + # via + # apache-beam + # google-auth +certifi==2023.7.22 + # via requests +cffi==1.15.1 + # via argon2-cffi-bindings +charset-normalizer==3.2.0 + # via requests +cloudpickle==2.2.1 + # via apache-beam +comm==0.1.4 + # via ipywidgets +crcmod==1.7 + # via apache-beam +decorator==5.1.1 + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert +dill==0.3.1.1 + # via apache-beam +docopt==0.6.2 + # via hdfs +entrypoints==0.4 + # via nbconvert +fastavro==1.8.3 + # via apache-beam +fasteners==0.19 + # via + # apache-beam + # google-apitools +flatbuffers==23.5.26 + # via tensorflow +gast==0.4.0 + # via tensorflow +gcsfs==0.2.3 + # via -r - +google-api-core[grpc]==2.11.1 + # via + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-core + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-language + # google-cloud-pubsub + # google-cloud-pubsublite + # google-cloud-recommendations-ai + # google-cloud-spanner + # google-cloud-videointelligence + # google-cloud-vision +google-api-python-client==1.7.12 + # via + # -r - + # tfx-bsl +google-apitools==0.5.31 + # via apache-beam +google-auth==2.23.0 + # via + # apache-beam + # gcsfs + # google-api-core + # google-api-python-client + # google-auth-httplib2 + # google-auth-oauthlib + # google-cloud-core + # tensorboard +google-auth-httplib2==0.1.1 + # via + # apache-beam + # google-api-python-client +google-auth-oauthlib==0.4.6 + # via + # gcsfs + # tensorboard +google-cloud-bigquery==3.11.4 + # via apache-beam +google-cloud-bigquery-storage==2.16.2 + # via apache-beam +google-cloud-bigtable==1.7.3 + # via apache-beam +google-cloud-core==2.3.3 + # via + # apache-beam + # google-cloud-bigquery + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-spanner +google-cloud-datastore==1.15.5 + # via apache-beam +google-cloud-dlp==3.12.3 + # via apache-beam +google-cloud-language==1.3.2 + # via apache-beam +google-cloud-pubsub==2.18.4 + # via + # apache-beam + # google-cloud-pubsublite +google-cloud-pubsublite==1.8.3 + # via apache-beam +google-cloud-recommendations-ai==0.7.1 + # via apache-beam +google-cloud-spanner==3.40.1 + # via apache-beam +google-cloud-videointelligence==1.16.3 + # via apache-beam +google-cloud-vision==3.4.4 + # via apache-beam +google-crc32c==1.5.0 + # via google-resumable-media +google-pasta==0.2.0 + # via tensorflow +google-resumable-media==2.6.0 + # via google-cloud-bigquery +googleapis-common-protos[grpc]==1.60.0 + # via + # google-api-core + # grpc-google-iam-v1 + # grpcio-status + # tensorflow-metadata +grpc-google-iam-v1==0.12.6 + # via + # google-cloud-bigtable + # google-cloud-pubsub + # google-cloud-spanner +grpcio==1.58.0 + # via + # apache-beam + # google-api-core + # google-cloud-bigquery + # google-cloud-pubsub + # google-cloud-pubsublite + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status + # tensorboard + # tensorflow + # tensorflow-serving-api +grpcio-status==1.48.2 + # via + # google-api-core + # google-cloud-pubsub + # google-cloud-pubsublite +h5py==3.9.0 + # via tensorflow +hdfs==2.7.2 + # via apache-beam +httplib2==0.21.0 + # via + # apache-beam + # google-api-python-client + # google-apitools + # google-auth-httplib2 + # oauth2client +idna==3.4 + # via requests +importlib-metadata==6.8.0 + # via markdown +importlib-resources==6.1.0 + # via + # jsonschema + # jsonschema-specifications +ipykernel==5.1.1 + # via + # -r - + # notebook +ipython==7.12.0 + # via + # -r - + # ipykernel + # ipywidgets + # tensorflow-model-analysis +ipython-genutils==0.2.0 + # via + # ipywidgets + # nbformat + # notebook +ipywidgets==7.8.1 + # via tensorflow-model-analysis +itables==0.1.0 + # via -r - +jedi==0.19.0 + # via ipython +jinja2==2.11.3 + # via + # -r - + # bokeh + # nbconvert + # notebook +joblib==0.14.1 + # via + # scikit-learn + # tensorflow-data-validation +jsonschema==4.19.1 + # via nbformat +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==5.3.5 + # via + # -r - + # ipykernel + # notebook +jupyter-core==5.3.1 + # via + # jupyter-client + # nbconvert + # nbformat + # notebook +jupyterlab-widgets==1.1.7 + # via ipywidgets +keras==2.10.0 + # via tensorflow +keras-preprocessing==1.1.2 + # via tensorflow +libclang==16.0.6 + # via tensorflow +markdown==3.4.4 + # via tensorboard +markupsafe==2.0.1 + # via + # -r - + # jinja2 +mistune==0.8.4 + # via + # -r - + # nbconvert +nbconvert==5.5.0 + # via + # -r - + # notebook +nbformat==4.4.0 + # via + # -r - + # nbconvert + # notebook +nest-asyncio==1.5.8 + # via notebook +notebook==6.4.13 + # via widgetsnbextension +numpy==1.24.4 + # via + # apache-beam + # bokeh + # h5py + # keras-preprocessing + # opt-einsum + # pandas + # pyarrow + # scikit-learn + # scipy + # tensorboard + # tensorflow + # tensorflow-data-validation + # tensorflow-model-analysis + # tfx-bsl +oauth2client==4.1.3 + # via google-apitools +oauthlib==3.2.2 + # via requests-oauthlib +objsize==0.6.1 + # via apache-beam +opt-einsum==3.3.0 + # via tensorflow +orjson==3.9.7 + # via apache-beam +overrides==6.5.0 + # via google-cloud-pubsublite +packaging==23.1 + # via + # bokeh + # google-cloud-bigquery + # tensorflow +pandas==1.5.3 + # via + # itables + # tensorflow-data-validation + # tensorflow-model-analysis + # tfx-bsl +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pillow==10.0.1 + # via bokeh +pkgutil-resolve-name==1.3.10 + # via jsonschema +platformdirs==3.10.0 + # via jupyter-core +prometheus-client==0.17.1 + # via notebook +prompt-toolkit==3.0.39 + # via ipython +proto-plus==1.22.3 + # via + # apache-beam + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-dlp + # google-cloud-pubsub + # google-cloud-recommendations-ai + # google-cloud-spanner + # google-cloud-vision +protobuf==3.19.6 + # via + # apache-beam + # google-api-core + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-bigtable + # google-cloud-datastore + # google-cloud-dlp + # google-cloud-language + # google-cloud-pubsub + # google-cloud-recommendations-ai + # google-cloud-spanner + # google-cloud-videointelligence + # google-cloud-vision + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status + # proto-plus + # tensorboard + # tensorflow + # tensorflow-data-validation + # tensorflow-metadata + # tensorflow-model-analysis + # tensorflow-serving-api + # tfx-bsl +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pyarrow==5.0.0 + # via + # apache-beam + # tensorflow-data-validation + # tensorflow-model-analysis + # tfx-bsl +pyasn1==0.5.0 + # via + # oauth2client + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 + # via + # google-auth + # oauth2client +pycparser==2.21 + # via cffi +pydot==1.4.2 + # via apache-beam +pyfarmhash==0.3.2 + # via tensorflow-data-validation +pygments==2.16.1 + # via + # ipython + # nbconvert +pymongo==3.13.0 + # via apache-beam +pyparsing==3.1.1 + # via + # httplib2 + # pydot +python-dateutil==2.8.2 + # via + # apache-beam + # bokeh + # google-cloud-bigquery + # jupyter-client + # pandas +pytz==2023.3.post1 + # via + # apache-beam + # pandas +pyyaml==6.0.1 + # via bokeh +pyzmq==25.1.1 + # via + # jupyter-client + # notebook +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications +regex==2023.8.8 + # via apache-beam +requests==2.31.0 + # via + # apache-beam + # gcsfs + # google-api-core + # google-cloud-bigquery + # hdfs + # requests-oauthlib + # tensorboard +requests-oauthlib==1.3.1 + # via google-auth-oauthlib +rpds-py==0.10.3 + # via + # jsonschema + # referencing +rsa==4.9 + # via + # google-auth + # oauth2client +scikit-learn==0.24.2 + # via -r - +scipy==1.10.1 + # via + # scikit-learn + # tensorflow-model-analysis +send2trash==1.8.2 + # via notebook +six==1.16.0 + # via + # astunparse + # bleach + # bokeh + # google-api-python-client + # google-apitools + # google-pasta + # hdfs + # keras-preprocessing + # oauth2client + # python-dateutil + # tensorflow + # tensorflow-data-validation + # tensorflow-model-analysis +sqlparse==0.4.4 + # via google-cloud-spanner +tensorboard==2.10.1 + # via tensorflow +tensorboard-data-server==0.6.1 + # via tensorboard +tensorboard-plugin-wit==1.8.1 + # via tensorboard +tensorflow==2.10.1 + # via + # -r - + # tensorflow-data-validation + # tensorflow-model-analysis + # tensorflow-serving-api + # tfx-bsl +tensorflow-data-validation==1.9.0 + # via -r - +tensorflow-estimator==2.10.0 + # via tensorflow +tensorflow-io-gcs-filesystem==0.34.0 + # via tensorflow +tensorflow-metadata==1.9.0 + # via + # -r - + # tensorflow-data-validation + # tensorflow-model-analysis + # tfx-bsl +tensorflow-model-analysis==0.40.0 + # via -r - +tensorflow-serving-api==2.10.1 + # via + # -r - + # tfx-bsl +termcolor==2.3.0 + # via tensorflow +terminado==0.17.1 + # via notebook +testpath==0.6.0 + # via nbconvert +tfx-bsl==1.9.0 + # via + # tensorflow-data-validation + # tensorflow-model-analysis +threadpoolctl==3.2.0 + # via scikit-learn +tornado==6.3.3 + # via + # -r - + # bokeh + # ipykernel + # jupyter-client + # notebook + # terminado +traitlets==5.10.0 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # nbconvert + # nbformat + # notebook +typing-extensions==4.8.0 + # via + # apache-beam + # tensorflow +uritemplate==3.0.1 + # via google-api-python-client +urllib3==1.26.16 + # via + # google-auth + # requests +wcwidth==0.2.6 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach +werkzeug==2.1.2 + # via tensorboard +wheel==0.41.2 + # via + # astunparse + # tensorboard +widgetsnbextension==3.6.6 + # via ipywidgets +wrapt==1.15.0 + # via tensorflow +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources +zstandard==0.21.0 + # via apache-beam # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/backend/src/apiserver/visualization/update_requirements.sh b/backend/src/apiserver/visualization/update_requirements.sh index 5dfbd8f12f..a1d22db67a 100755 --- a/backend/src/apiserver/visualization/update_requirements.sh +++ b/backend/src/apiserver/visualization/update_requirements.sh @@ -1,9 +1,9 @@ #!/bin/bash # This image should be in sync with Dockerfile.visualization. -IMAGE="tensorflow/tensorflow:2.5.1" +IMAGE="tensorflow/tensorflow:2.10.1" # tensorflow/tfx default entrypoint is Apache BEAM, because Apache BEAM doesn't # support custom entrypoint for now. We need to override with --entrypoint "" # for other `docker run` usecase. # https://github.com/tensorflow/tfx/blob/master/tfx/tools/docker/Dockerfile#L71 -../../../update_requirements.sh $IMAGE requirements.txt +../../../../hack/update-requirements.sh $IMAGE requirements.txt diff --git a/backend/src/common/util/execution_status.go b/backend/src/common/util/execution_status.go index 7eff8a2064..6831e141f4 100644 --- a/backend/src/common/util/execution_status.go +++ b/backend/src/common/util/execution_status.go @@ -31,7 +31,7 @@ type NodeStatus struct { Children []string } -type RetrieveArtifact func(request *api.ReadArtifactRequest, user string) (*api.ReadArtifactResponse, error) +type RetrieveArtifact func(request *api.ReadArtifactRequest) (*api.ReadArtifactResponse, error) // Abstract interface to encapsulate the resources of the execution runtime specifically // for status information. This interface is mainly to access the status related information @@ -61,7 +61,7 @@ type ExecutionStatus interface { // This function was in metrics_reporter.go. Moved to here because it // accesses the orchestration engine specific data struct. encapsulate the // specific data struct and provide a abstract function here. - CollectionMetrics(retrieveArtifact RetrieveArtifact, user string) ([]*api.RunMetric, []error) + CollectionMetrics(retrieveArtifact RetrieveArtifact) ([]*api.RunMetric, []error) // does ExecutionStatus contain any finished node or not HasMetrics() bool diff --git a/backend/src/common/util/metrics.go b/backend/src/common/util/metrics.go new file mode 100644 index 0000000000..dc3ae8898c --- /dev/null +++ b/backend/src/common/util/metrics.go @@ -0,0 +1,60 @@ +// Copyright 2018 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +type MetricsChan chan prometheus.Metric + +// GetMetricValue get metric value from registered Collector +func GetMetricValue(collector prometheus.Collector) float64 { + var total float64 + collectValue(collector, func(m dto.Metric) { + // retrieves data if current collector is a gauge + // if not then retrieves from a counter + if gauge := m.GetGauge(); gauge != nil { + total += m.GetGauge().GetValue() + } else { + if counter := m.GetCounter(); counter != nil { + total += m.GetCounter().GetValue() + } else { + glog.Errorln("invalid type, only valid collectors are: gauge, counter") + total = 0 + } + } + }) + return total +} + +func collectValue(collector prometheus.Collector, do func(dto.Metric)) { + c := make(MetricsChan) + + // collect calls the function for each metric associated with the Collector + go func(c MetricsChan) { + collector.Collect(c) + close(c) + }(c) + + // range across distinct label vector values + for x := range c { + m := dto.Metric{} + _ = x.Write(&m) + do(m) + } +} diff --git a/backend/src/common/util/pipelinerun.go b/backend/src/common/util/pipelinerun.go index a2b97b2acb..4b6c7789b8 100644 --- a/backend/src/common/util/pipelinerun.go +++ b/backend/src/common/util/pipelinerun.go @@ -655,12 +655,12 @@ func (pr *PipelineRun) GenerateRetryExecution() (ExecutionSpec, []string, error) return NewPipelineRun(newWF), podsToDelete, nil } -func (pr *PipelineRun) CollectionMetrics(retrieveArtifact RetrieveArtifact, user string) ([]*api.RunMetric, []error) { +func (pr *PipelineRun) CollectionMetrics(retrieveArtifact RetrieveArtifact) ([]*api.RunMetric, []error) { runID := pr.ObjectMeta.Labels[LabelKeyWorkflowRunId] runMetrics := []*api.RunMetric{} partialFailures := []error{} for _, taskrunStatus := range pr.Status.PipelineRunStatusFields.TaskRuns { - nodeMetrics, err := collectTaskRunMetricsOrNil(runID, *taskrunStatus, retrieveArtifact, user) + nodeMetrics, err := collectTaskRunMetricsOrNil(runID, *taskrunStatus, retrieveArtifact) if err != nil { partialFailures = append(partialFailures, err) continue @@ -680,7 +680,7 @@ func (pr *PipelineRun) CollectionMetrics(retrieveArtifact RetrieveArtifact, user } func collectTaskRunMetricsOrNil( - runID string, taskrunStatus pipelineapi.PipelineRunTaskRunStatus, retrieveArtifact RetrieveArtifact, user string) ( + runID string, taskrunStatus pipelineapi.PipelineRunTaskRunStatus, retrieveArtifact RetrieveArtifact) ( []*api.RunMetric, error) { defer func() { @@ -692,7 +692,7 @@ func collectTaskRunMetricsOrNil( taskrunStatus.Status.TaskRunStatusFields.CompletionTime == nil { return nil, nil } - metricsJSON, err := readTaskRunMetricsJSONOrEmpty(runID, taskrunStatus, retrieveArtifact, user) + metricsJSON, err := readTaskRunMetricsJSONOrEmpty(runID, taskrunStatus, retrieveArtifact) if err != nil || metricsJSON == "" { return nil, err } @@ -726,7 +726,7 @@ func collectTaskRunMetricsOrNil( func readTaskRunMetricsJSONOrEmpty( runID string, nodeStatus pipelineapi.PipelineRunTaskRunStatus, - retrieveArtifact RetrieveArtifact, user string) (string, error) { + retrieveArtifact RetrieveArtifact) (string, error) { // Tekton doesn't support any artifact spec, artifact records are done by our custom metadata writers: // if nodeStatus.Outputs == nil || nodeStatus.Outputs.Artifacts == nil { // return "", nil // No output artifacts, skip the reporting @@ -746,7 +746,7 @@ func readTaskRunMetricsJSONOrEmpty( NodeId: nodeStatus.PipelineTaskName, ArtifactName: metricsArtifactName, } - artifactResponse, err := retrieveArtifact(artifactRequest, user) + artifactResponse, err := retrieveArtifact(artifactRequest) if err != nil { return "", err } diff --git a/backend/src/common/util/scheduled_workflow.go b/backend/src/common/util/scheduled_workflow.go index 07b292d719..caa25dd3cb 100644 --- a/backend/src/common/util/scheduled_workflow.go +++ b/backend/src/common/util/scheduled_workflow.go @@ -31,9 +31,9 @@ const ( SWFlegacy ScheduledWorkflowType = "legacy" SWFunknown ScheduledWorkflowType = "Unknown" - apiVersionV1 = "kubeflow.org/v1beta1" - apiVersionV2 = "kubeflow.org/v2beta1" - swfKind = "ScheduledWorkflow" + ApiVersionV1 = "kubeflow.org/v1beta1" + ApiVersionV2 = "kubeflow.org/v2beta1" + SwfKind = "ScheduledWorkflow" ) // ScheduledWorkflow is a type to help manipulate ScheduledWorkflow objects. @@ -160,9 +160,9 @@ func (s *ScheduledWorkflow) ToStringForStore() string { } func (s *ScheduledWorkflow) GetVersion() ScheduledWorkflowType { - if strings.HasPrefix(s.APIVersion, apiVersionV1) && s.Kind == swfKind { + if strings.HasPrefix(s.APIVersion, ApiVersionV1) && s.Kind == SwfKind { return SWFv1 - } else if strings.HasPrefix(s.APIVersion, apiVersionV2) && s.Kind == swfKind { + } else if strings.HasPrefix(s.APIVersion, ApiVersionV2) && s.Kind == SwfKind { return SWFv2 } else if s.APIVersion == "" && s.Kind == "" { return SWFlegacy diff --git a/backend/src/common/util/workflow.go b/backend/src/common/util/workflow.go index b898a9445f..5d9b4e69bf 100644 --- a/backend/src/common/util/workflow.go +++ b/backend/src/common/util/workflow.go @@ -436,12 +436,12 @@ const ( maxMetricsCountLimit = 50 ) -func (w *Workflow) CollectionMetrics(retrieveArtifact RetrieveArtifact, user string) ([]*api.RunMetric, []error) { +func (w *Workflow) CollectionMetrics(retrieveArtifact RetrieveArtifact) ([]*api.RunMetric, []error) { runID := w.Labels[LabelKeyWorkflowRunId] runMetrics := make([]*api.RunMetric, 0, len(w.Status.Nodes)) partialFailures := make([]error, 0, len(w.Status.Nodes)) for _, nodeStatus := range w.Status.Nodes { - nodeMetrics, err := collectNodeMetricsOrNil(runID, &nodeStatus, retrieveArtifact, user) + nodeMetrics, err := collectNodeMetricsOrNil(runID, &nodeStatus, retrieveArtifact) if err != nil { partialFailures = append(partialFailures, err) continue @@ -460,13 +460,13 @@ func (w *Workflow) CollectionMetrics(retrieveArtifact RetrieveArtifact, user str return runMetrics, partialFailures } -func collectNodeMetricsOrNil(runID string, nodeStatus *workflowapi.NodeStatus, retrieveArtifact RetrieveArtifact, user string) ( +func collectNodeMetricsOrNil(runID string, nodeStatus *workflowapi.NodeStatus, retrieveArtifact RetrieveArtifact) ( []*api.RunMetric, error, ) { if !nodeStatus.Completed() { return nil, nil } - metricsJSON, err := readNodeMetricsJSONOrEmpty(runID, nodeStatus, retrieveArtifact, user) + metricsJSON, err := readNodeMetricsJSONOrEmpty(runID, nodeStatus, retrieveArtifact) if err != nil || metricsJSON == "" { return nil, err } @@ -499,7 +499,7 @@ func collectNodeMetricsOrNil(runID string, nodeStatus *workflowapi.NodeStatus, r } func readNodeMetricsJSONOrEmpty(runID string, nodeStatus *workflowapi.NodeStatus, - retrieveArtifact RetrieveArtifact, user string, + retrieveArtifact RetrieveArtifact, ) (string, error) { if nodeStatus.Outputs == nil || nodeStatus.Outputs.Artifacts == nil { return "", nil // No output artifacts, skip the reporting @@ -520,7 +520,7 @@ func readNodeMetricsJSONOrEmpty(runID string, nodeStatus *workflowapi.NodeStatus NodeId: nodeStatus.ID, ArtifactName: metricsArtifactName, } - artifactResponse, err := retrieveArtifact(artifactRequest, user) + artifactResponse, err := retrieveArtifact(artifactRequest) if err != nil { return "", err } diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index 3e4f33842d..d7c488972a 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -116,8 +116,8 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S wf: wf, templates: make(map[string]*wfapi.Template), // TODO(chensun): release process and update the images. - driverImage: "gcr.io/ml-pipeline/kfp-driver@sha256:0ce9bf20ac9cbb21e84ff0762d5ae508d21e9c85fde2b14b51363bd1b8cd7528", - launcherImage: "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4", + driverImage: "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707", + launcherImage: "gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e", job: job, spec: spec, executors: deploy.GetExecutors(), diff --git a/backend/src/v2/compiler/tektoncompiler/tekton.go b/backend/src/v2/compiler/tektoncompiler/tekton.go index 69e1ac7a9a..19f95ecb93 100644 --- a/backend/src/v2/compiler/tektoncompiler/tekton.go +++ b/backend/src/v2/compiler/tektoncompiler/tekton.go @@ -113,7 +113,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S c := &pipelinerunCompiler{ pr: pr, // TODO(chensun): release process and update the images. - launcherImage: "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4", + launcherImage: "gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e", job: job, spec: spec, dagStack: make([]string, 0, 10), diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml index cdc2640f29..f493aa9eed 100755 --- a/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/exit_handler.yaml @@ -99,7 +99,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -276,7 +276,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -434,7 +434,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -560,8 +560,8 @@ spec: - name: mlmd_server_port value: "8080" runAfter: - - fail-op - print-op-2 + - fail-op taskRef: apiVersion: kfp-driver.tekton.dev/v1alpha1 kind: KFPDriver diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml index bd76c05f42..482fa01f18 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/hello_world.yaml @@ -81,7 +81,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml index 2002b17ae2..57834289e2 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/importer.yaml @@ -87,7 +87,7 @@ spec: value: metadata-grpc-service.kubeflow.svc.cluster.local - name: METADATA_GRPC_SERVICE_PORT value: "8080" - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e name: importer-main resources: {} - name: root-system-dag-driver diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml index 383bac891d..fac4c29d19 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/loop_static.yaml @@ -154,7 +154,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -313,7 +313,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -514,7 +514,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml index 4b1c7be4f0..e30afd4657 100755 --- a/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/mnist_pipeline.yaml @@ -89,7 +89,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -256,7 +256,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -420,7 +420,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -599,7 +599,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -928,7 +928,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -1119,7 +1119,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -1284,7 +1284,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -1457,7 +1457,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -1630,7 +1630,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} diff --git a/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml b/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml index 8a6ad9f8ca..6b4f310e62 100644 --- a/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml +++ b/backend/src/v2/compiler/tektoncompiler/testdata/nestedloop.yaml @@ -86,7 +86,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -326,7 +326,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -538,7 +538,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} @@ -751,7 +751,7 @@ spec: - launcher-v2 - --copy - /tekton/home/launch - image: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + image: gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e imagePullPolicy: Always name: kfp-launcher resources: {} diff --git a/backend/src/v2/component/launcher_v2.go b/backend/src/v2/component/launcher_v2.go index e0f5060114..92a951c1c1 100644 --- a/backend/src/v2/component/launcher_v2.go +++ b/backend/src/v2/component/launcher_v2.go @@ -714,7 +714,7 @@ func localPathForURI(uri string) (string, error) { func prepareOutputFolders(executorInput *pipelinespec.ExecutorInput) error { for name, parameter := range executorInput.GetOutputs().GetParameters() { dir := filepath.Dir(parameter.OutputFile) - if err := os.MkdirAll(dir, 0644); err != nil { + if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create directory %q for output parameter %q: %w", dir, name, err) } } @@ -730,7 +730,7 @@ func prepareOutputFolders(executorInput *pipelinespec.ExecutorInput) error { return fmt.Errorf("failed to generate local storage path for output artifact %q: %w", name, err) } - if err := os.MkdirAll(filepath.Dir(localPath), 0644); err != nil { + if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil { return fmt.Errorf("unable to create directory %q for output artifact %q: %w", filepath.Dir(localPath), name, err) } } diff --git a/backend/src/v2/driver/driver.go b/backend/src/v2/driver/driver.go index eda53baad5..d227855ca3 100644 --- a/backend/src/v2/driver/driver.go +++ b/backend/src/v2/driver/driver.go @@ -768,7 +768,11 @@ func resolveInputs(ctx context.Context, dag *metadata.DAG, iterationIndex *int, if err != nil { return nil, err } - glog.Infof("parent DAG input parameters %+v", inputParams) + inputArtifacts, err := mlmd.GetInputArtifactsByExecutionID(ctx, dag.Execution.GetID()) + if err != nil { + return nil, err + } + glog.Infof("parent DAG input parameters: %+v, artifacts: %+v", inputParams, inputArtifacts) inputs = &pipelinespec.ExecutorInput_Inputs{ ParameterValues: make(map[string]*structpb.Value), Artifacts: make(map[string]*pipelinespec.ArtifactList), @@ -998,7 +1002,15 @@ func resolveInputs(ctx context.Context, dag *metadata.DAG, iterationIndex *int, } switch t := artifactSpec.Kind.(type) { case *pipelinespec.TaskInputsSpec_InputArtifactSpec_ComponentInputArtifact: - return nil, artifactError(fmt.Errorf("component input artifact not implemented yet")) + inputArtifactName := artifactSpec.GetComponentInputArtifact() + if inputArtifactName == "" { + return nil, artifactError(fmt.Errorf("component input artifact key is empty")) + } + v, ok := inputArtifacts[inputArtifactName] + if !ok { + return nil, artifactError(fmt.Errorf("parent DAG does not have input artifact %s", inputArtifactName)) + } + inputs.Artifacts[name] = v case *pipelinespec.TaskInputsSpec_InputArtifactSpec_TaskOutputArtifact: taskOutput := artifactSpec.GetTaskOutputArtifact() diff --git a/backend/src/v2/test/requirements.txt b/backend/src/v2/test/requirements.txt index 7c9f5ac2b6..6eece4be3a 100644 --- a/backend/src/v2/test/requirements.txt +++ b/backend/src/v2/test/requirements.txt @@ -2,7 +2,7 @@ -e ../../../../sdk/python # TODO(chensun): remove the deprecated dependencies once migrated tests. -r ../../../../sdk/python/requirements-deprecated.txt -ml-metadata==1.5.0 +ml-metadata==1.14.0 minio==7.0.4 google-cloud-storage fire diff --git a/backend/test/integration/README.md b/backend/test/integration/README.md index ee6e510971..d87a330310 100644 --- a/backend/test/integration/README.md +++ b/backend/test/integration/README.md @@ -5,7 +5,22 @@ ### How to run +The default integration test will test the default Database, MySQL. + 1. Configure kubectl to connect to your kfp cluster. 2. Run the following for all integration tests: `NAMESPACE= ./run_tests_locally.sh`. 3. Or run the following to select certain tests: `NAMESPACE= ./run_tests_locally.sh -testify.m Job`. Reference: https://stackoverflow.com/a/43312451 + +### Run database tests with PostgreSQL + +To run this test, you need to first deploy the PostgreSQL images on your Kubernetes cluster. For how to deploy, +see [instructions here](../../../manifests/kustomize/third-party/postgresql/README.md). + +When testing against postgreSQL, all integration tests with MySQL will be disabled. Use an argument `postgres` to run +test against a PostgreSQL database: +``` +NAMESPACE= ./run_tests_locally.sh postgres +``` + + diff --git a/backend/test/integration/db_test.go b/backend/test/integration/db_test.go new file mode 100644 index 0000000000..be65889eff --- /dev/null +++ b/backend/test/integration/db_test.go @@ -0,0 +1,79 @@ +// Copyright 2023 The Kubeflow Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integration + +import ( + "testing" + "time" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + _ "github.com/jackc/pgx/v5/stdlib" + cm "github.com/kubeflow/pipelines/backend/src/apiserver/client_manager" +) + +type DBTestSuite struct { + suite.Suite +} + +// Skip if it's not integration test running. +func (s *DBTestSuite) SetupTest() { + if !*runIntegrationTests { + s.T().SkipNow() + return + } +} + +// Test MySQL initializes correctly +func (s *DBTestSuite) TestInitDBClient_MySQL() { + if *runPostgreSQLTests { + s.T().SkipNow() + return + } + t := s.T() + viper.Set("DBDriverName", "mysql") + viper.Set("DBConfig.MySQLConfig.DBName", "mlpipeline") + // The default port-forwarding IP address that test uses is different compared to production + if *localTest { + viper.Set("DBConfig.MySQLConfig.Host", "localhost") + } + duration, _ := time.ParseDuration("1m") + db := cm.InitDBClient(duration) + assert.NotNil(t, db) +} + +// Test PostgreSQL initializes correctly +func (s *DBTestSuite) TestInitDBClient_PostgreSQL() { + if !*runPostgreSQLTests { + s.T().SkipNow() + return + } + t := s.T() + viper.Set("DBDriverName", "pgx") + viper.Set("DBConfig.PostgreSQLConfig.DBName", "mlpipeline") + // The default port-forwarding IP address that test uses is different compared to production + viper.Set("DBConfig.PostgreSQLConfig.Host", "127.0.0.3") + viper.Set("DBConfig.PostgreSQLConfig.User", "user") + viper.Set("DBConfig.PostgreSQLConfig.Password", "password") + duration, _ := time.ParseDuration("1m") + db := cm.InitDBClient(duration) + assert.NotNil(t, db) +} + +func TestDB(t *testing.T) { + suite.Run(t, new(DBTestSuite)) +} diff --git a/backend/test/integration/flags.go b/backend/test/integration/flags.go index 7585d2d75c..95af0e0d55 100644 --- a/backend/test/integration/flags.go +++ b/backend/test/integration/flags.go @@ -24,6 +24,8 @@ var ( initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization") runIntegrationTests = flag.Bool("runIntegrationTests", false, "Whether to also run integration tests that call the service") runUpgradeTests = flag.Bool("runUpgradeTests", false, "Whether to run upgrade tests") + runPostgreSQLTests = flag.Bool("runPostgreSQLTests", false, "Run integration test with PostgreSQL") + localTest = flag.Bool("localTest", false, "Run integration test locally") ) /** diff --git a/backend/test/integration/run_tests_locally.sh b/backend/test/integration/run_tests_locally.sh index 8dfe095e00..371ad83f4e 100755 --- a/backend/test/integration/run_tests_locally.sh +++ b/backend/test/integration/run_tests_locally.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -e +set -ex if [ -z "${NAMESPACE}" ]; then echo "NAMESPACE env var is not provided, please set it to your KFP namespace" @@ -32,7 +32,26 @@ case "$response" in ;; esac +function cleanup() { + echo "killing kubectl port forward before exit" + kill "$PORT_FORWARD_PID" +} +trap cleanup EXIT + echo "Starting integration tests..." -command="go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true -isDevMode=true" -echo $command "$@" + +if [ "$1" == "postgres" ]; then + echo "Starting PostgreSQL DB port forwarding..." + kubectl -n "$NAMESPACE" port-forward svc/postgres-service 5432:5432 --address="127.0.0.3" & PORT_FORWARD_PID=$! + # wait for kubectl port forward + sleep 10 + command="go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true -isDevMode=true -runPostgreSQLTests=true -localTest=true" +else + echo "Starting MySQL DB port forwarding..." + kubectl -n "$NAMESPACE" port-forward svc/mysql 3306:3306 --address=localhost & PORT_FORWARD_PID=$! + # wait for kubectl port forward + sleep 10 + command="go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true -isDevMode=true -localTest=true" +fi + $command "$@" diff --git a/backend/third_party_licenses/apiserver.csv b/backend/third_party_licenses/apiserver.csv index f591c1cfed..0a7caeb014 100644 --- a/backend/third_party_licenses/apiserver.csv +++ b/backend/third_party_licenses/apiserver.csv @@ -73,7 +73,7 @@ github.com/klauspost/pgzip,https://github.com/klauspost/pgzip/blob/v1.2.5/LICENS github.com/kubeflow/kfp-tekton/tekton-catalog/pipeline-loops/pkg/apis/pipelineloop,https://github.com/kubeflow/kfp-tekton/blob/2dc9314d8d35/tekton-catalog/pipeline-loops/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/758c91f76784/api/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 -github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e78ed557ddcb/third_party/ml-metadata/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 github.com/lann/builder,https://github.com/lann/builder/blob/47ae307949d0/LICENSE,MIT github.com/lann/ps,https://github.com/lann/ps/blob/62de8c46ede0/LICENSE,MIT github.com/lestrrat-go/strftime,https://github.com/lestrrat-go/strftime/blob/v1.0.4/LICENSE,MIT @@ -98,7 +98,7 @@ github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/ github.com/pelletier/go-toml,https://github.com/pelletier/go-toml/blob/v1.9.5/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -122,13 +122,13 @@ go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MI go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0 -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/04be3eba:LICENSE,BSD-3-Clause diff --git a/backend/third_party_licenses/cache_server.csv b/backend/third_party_licenses/cache_server.csv index a66807349d..c43ee7e1f8 100644 --- a/backend/third_party_licenses/cache_server.csv +++ b/backend/third_party_licenses/cache_server.csv @@ -70,12 +70,20 @@ github.com/oliveagle/jsonpath,https://github.com/oliveagle/jsonpath/blob/2e52cf6 github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 github.com/peterhellberg/duration,https://github.com/peterhellberg/duration/blob/ec6baeebcd10/LICENSE,MIT github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause +<<<<<<< HEAD github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 github.com/prometheus/statsd_exporter/pkg/mapper,https://github.com/prometheus/statsd_exporter/blob/v0.21.0/LICENSE,Apache-2.0 +======= +github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.12.1/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 +github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.32.1/LICENSE,Apache-2.0 +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.32.1/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause +github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.7.3/LICENSE,Apache-2.0 +>>>>>>> kfp/master github.com/robfig/cron/v3,https://github.com/robfig/cron/blob/v3.0.1/LICENSE,MIT github.com/shopspring/decimal,https://github.com/shopspring/decimal/blob/v1.2.0/LICENSE,MIT github.com/sirupsen/logrus,https://github.com/sirupsen/logrus/blob/v1.9.0/LICENSE,MIT @@ -84,6 +92,7 @@ github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3- github.com/tektoncd/pipeline/pkg,https://github.com/tektoncd/pipeline/blob/v0.47.3/LICENSE,Apache-2.0 github.com/valyala/bytebufferpool,https://github.com/valyala/bytebufferpool/blob/v1.0.0/LICENSE,MIT github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1.2.1/LICENSE,MIT +<<<<<<< HEAD go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.7.5/LICENSE,Apache-2.0 go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE,Apache-2.0 go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MIT @@ -102,6 +111,18 @@ gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2 google.golang.org/api/support/bundler,https://github.com/googleapis/google-api-go-client/blob/v0.116.0/LICENSE,BSD-3-Clause google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/dcfb400f0633/LICENSE,Apache-2.0 google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.54.0/LICENSE,Apache-2.0 +======= +go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.8.2/LICENSE,Apache-2.0 +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/d3ed0bb2:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/90d013bb:LICENSE,BSD-3-Clause +google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/1973136f34c6/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE,Apache-2.0 +>>>>>>> kfp/master google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.30.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause gopkg.in/jcmturner/aescts.v1,https://github.com/jcmturner/aescts/blob/v1.0.1/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/driver.csv b/backend/third_party_licenses/driver.csv new file mode 100644 index 0000000000..b05a884c4c --- /dev/null +++ b/backend/third_party_licenses/driver.csv @@ -0,0 +1,69 @@ +cloud.google.com/go/compute/metadata,https://github.com/googleapis/google-cloud-go/blob/compute/v1.3.0/compute/LICENSE,Apache-2.0 +cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v0.1.1/iam/LICENSE,Apache-2.0 +cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.100.2/LICENSE,Apache-2.0 +cloud.google.com/go/storage,https://github.com/googleapis/google-cloud-go/blob/storage/v1.20.0/storage/LICENSE,Apache-2.0 +github.com/PuerkitoBio/purell,https://github.com/PuerkitoBio/purell/blob/v1.1.1/LICENSE,BSD-3-Clause +github.com/PuerkitoBio/urlesc,https://github.com/PuerkitoBio/urlesc/blob/de5bf2ad4578/LICENSE,BSD-3-Clause +github.com/antlr/antlr4/runtime/Go/antlr,https://github.com/antlr/antlr4/blob/b48c857c3a0e/runtime/Go/antlr/LICENSE,BSD-3-Clause +github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.42.50/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.42.50/internal/sync/singleflight/LICENSE,BSD-3-Clause +github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC +github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.8.0/LICENSE,MIT +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.19.6/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.19.15/LICENSE,Apache-2.0 +github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause +github.com/golang/glog,https://github.com/golang/glog/blob/v1.0.0/LICENSE,Apache-2.0 +github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 +github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.2/LICENSE,BSD-3-Clause +github.com/google/cel-go,https://github.com/google/cel-go/blob/v0.9.0/LICENSE,Apache-2.0 +github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.5.7-v3refs/LICENSE,Apache-2.0 +github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.5.7/LICENSE,BSD-3-Clause +github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 +github.com/google/uuid,https://github.com/google/uuid/blob/v1.3.0/LICENSE,BSD-3-Clause +github.com/google/wire,https://github.com/google/wire/blob/v0.4.0/LICENSE,Apache-2.0 +github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.1.1/v2/LICENSE,BSD-3-Clause +github.com/grpc-ecosystem/go-grpc-middleware,https://github.com/grpc-ecosystem/go-grpc-middleware/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause +github.com/jmespath/go-jmespath,https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE,Apache-2.0 +github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT +github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT +github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/758c91f76784/api/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/bd9f74e34de6/kubernetes_platform/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 +github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT +github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 +github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 +github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause +github.com/stoewer/go-strcase,https://github.com/stoewer/go-strcase/blob/v1.2.0/LICENSE,MIT +go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.23.0/LICENSE,Apache-2.0 +gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0 +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/d3ed0bb2:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/90d013bb:LICENSE,BSD-3-Clause +golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/5ec99f83:LICENSE,BSD-3-Clause +google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.70.0/LICENSE,BSD-3-Clause +google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.70.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause +google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/1973136f34c6/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.30.0/LICENSE,BSD-3-Clause +gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause +gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 +gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT +k8s.io/api,https://github.com/kubernetes/api/blob/v0.24.3/LICENSE,Apache-2.0 +k8s.io/apimachinery/pkg,https://github.com/kubernetes/apimachinery/blob/v0.24.3/LICENSE,Apache-2.0 +k8s.io/apimachinery/third_party/forked/golang/reflect,https://github.com/kubernetes/apimachinery/blob/v0.24.3/third_party/forked/golang/LICENSE,BSD-3-Clause +k8s.io/client-go,https://github.com/kubernetes/client-go/blob/v0.24.3/LICENSE,Apache-2.0 +k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.60.1/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg/validation/spec,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/pkg/validation/spec/LICENSE,Apache-2.0 +k8s.io/utils,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/LICENSE,Apache-2.0 +k8s.io/utils/internal/third_party/forked/golang/net,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/internal/third_party/forked/golang/LICENSE,BSD-3-Clause +sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/9f7c6b3444d2/LICENSE,Apache-2.0 +sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.2.1/LICENSE,Apache-2.0 +sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.3.0/LICENSE,MIT diff --git a/backend/third_party_licenses/launcher.csv b/backend/third_party_licenses/launcher.csv new file mode 100644 index 0000000000..4aba0f16d3 --- /dev/null +++ b/backend/third_party_licenses/launcher.csv @@ -0,0 +1,64 @@ +cloud.google.com/go/compute/metadata,https://github.com/googleapis/google-cloud-go/blob/compute/v1.3.0/compute/LICENSE,Apache-2.0 +cloud.google.com/go/iam,https://github.com/googleapis/google-cloud-go/blob/iam/v0.1.1/iam/LICENSE,Apache-2.0 +cloud.google.com/go/internal,https://github.com/googleapis/google-cloud-go/blob/v0.100.2/LICENSE,Apache-2.0 +cloud.google.com/go/storage,https://github.com/googleapis/google-cloud-go/blob/storage/v1.20.0/storage/LICENSE,Apache-2.0 +github.com/PuerkitoBio/purell,https://github.com/PuerkitoBio/purell/blob/v1.1.1/LICENSE,BSD-3-Clause +github.com/PuerkitoBio/urlesc,https://github.com/PuerkitoBio/urlesc/blob/de5bf2ad4578/LICENSE,BSD-3-Clause +github.com/aws/aws-sdk-go,https://github.com/aws/aws-sdk-go/blob/v1.42.50/LICENSE.txt,Apache-2.0 +github.com/aws/aws-sdk-go/internal/sync/singleflight,https://github.com/aws/aws-sdk-go/blob/v1.42.50/internal/sync/singleflight/LICENSE,BSD-3-Clause +github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC +github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.8.0/LICENSE,MIT +github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v1.2.2/LICENSE,Apache-2.0 +github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE,Apache-2.0 +github.com/go-openapi/jsonreference,https://github.com/go-openapi/jsonreference/blob/v0.19.6/LICENSE,Apache-2.0 +github.com/go-openapi/swag,https://github.com/go-openapi/swag/blob/v0.19.15/LICENSE,Apache-2.0 +github.com/gogo/protobuf,https://github.com/gogo/protobuf/blob/v1.3.2/LICENSE,BSD-3-Clause +github.com/golang/glog,https://github.com/golang/glog/blob/v1.0.0/LICENSE,Apache-2.0 +github.com/golang/groupcache/lru,https://github.com/golang/groupcache/blob/41bb18bfe9da/LICENSE,Apache-2.0 +github.com/golang/protobuf,https://github.com/golang/protobuf/blob/v1.5.2/LICENSE,BSD-3-Clause +github.com/google/gnostic,https://github.com/google/gnostic/blob/v0.5.7-v3refs/LICENSE,Apache-2.0 +github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.5.7/LICENSE,BSD-3-Clause +github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.2.0/LICENSE,Apache-2.0 +github.com/google/wire,https://github.com/google/wire/blob/v0.4.0/LICENSE,Apache-2.0 +github.com/googleapis/gax-go/v2,https://github.com/googleapis/gax-go/blob/v2.1.1/v2/LICENSE,BSD-3-Clause +github.com/grpc-ecosystem/go-grpc-middleware,https://github.com/grpc-ecosystem/go-grpc-middleware/blob/v1.3.0/LICENSE,Apache-2.0 +github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause +github.com/jmespath/go-jmespath,https://github.com/jmespath/go-jmespath/blob/v0.4.0/LICENSE,Apache-2.0 +github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT +github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.12/LICENSE,MIT +github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/758c91f76784/api/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 +github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT +github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 +github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 +github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause +go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0.23.0/LICENSE,Apache-2.0 +gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0 +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/d3ed0bb2:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/90d013bb:LICENSE,BSD-3-Clause +golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/5ec99f83:LICENSE,BSD-3-Clause +google.golang.org/api,https://github.com/googleapis/google-api-go-client/blob/v0.70.0/LICENSE,BSD-3-Clause +google.golang.org/api/internal/third_party/uritemplates,https://github.com/googleapis/google-api-go-client/blob/v0.70.0/internal/third_party/uritemplates/LICENSE,BSD-3-Clause +google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/1973136f34c6/LICENSE,Apache-2.0 +google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE,Apache-2.0 +google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.30.0/LICENSE,BSD-3-Clause +gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause +gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 +gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE,MIT +k8s.io/api,https://github.com/kubernetes/api/blob/v0.24.3/LICENSE,Apache-2.0 +k8s.io/apimachinery/pkg,https://github.com/kubernetes/apimachinery/blob/v0.24.3/LICENSE,Apache-2.0 +k8s.io/apimachinery/third_party/forked/golang/reflect,https://github.com/kubernetes/apimachinery/blob/v0.24.3/third_party/forked/golang/LICENSE,BSD-3-Clause +k8s.io/client-go,https://github.com/kubernetes/client-go/blob/v0.24.3/LICENSE,Apache-2.0 +k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.60.1/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/LICENSE,Apache-2.0 +k8s.io/kube-openapi/pkg/validation/spec,https://github.com/kubernetes/kube-openapi/blob/011e075b9cb8/pkg/validation/spec/LICENSE,Apache-2.0 +k8s.io/utils,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/LICENSE,Apache-2.0 +k8s.io/utils/internal/third_party/forked/golang/net,https://github.com/kubernetes/utils/blob/3a6ce19ff2f9/internal/third_party/forked/golang/LICENSE,BSD-3-Clause +sigs.k8s.io/json,https://github.com/kubernetes-sigs/json/blob/9f7c6b3444d2/LICENSE,Apache-2.0 +sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.2.1/LICENSE,Apache-2.0 +sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.3.0/LICENSE,MIT diff --git a/backend/third_party_licenses/persistence_agent.csv b/backend/third_party_licenses/persistence_agent.csv index a65adef487..480e11448e 100644 --- a/backend/third_party_licenses/persistence_agent.csv +++ b/backend/third_party_licenses/persistence_agent.csv @@ -18,7 +18,6 @@ github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/L github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful/v3,https://github.com/emicklei/go-restful/blob/v3.9.0/LICENSE,MIT github.com/evanphx/json-patch/v5,https://github.com/evanphx/json-patch/blob/v5.6.0/v5/LICENSE,BSD-3-Clause -github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.5.4/LICENSE,BSD-3-Clause github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT github.com/go-kit/log,https://github.com/go-kit/log/blob/v0.2.0/LICENSE,MIT github.com/go-logfmt/logfmt,https://github.com/go-logfmt/logfmt/blob/v0.5.1/LICENSE,MIT @@ -45,7 +44,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2,https://github.com/grpc-ecosystem/grpc github.com/hashicorp/errwrap,https://github.com/hashicorp/errwrap/blob/v1.1.0/LICENSE,MPL-2.0 github.com/hashicorp/go-multierror,https://github.com/hashicorp/go-multierror/blob/v1.1.1/LICENSE,MPL-2.0 github.com/hashicorp/go-uuid,https://github.com/hashicorp/go-uuid/blob/v1.0.3/LICENSE,MPL-2.0 -github.com/hashicorp/hcl,https://github.com/hashicorp/hcl/blob/v1.0.0/LICENSE,MPL-2.0 github.com/huandu/xstrings,https://github.com/huandu/xstrings/blob/v1.3.2/LICENSE,MIT github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.13/LICENSE,BSD-3-Clause github.com/jcmturner/gofork,https://github.com/jcmturner/gofork/blob/v1.0.0/LICENSE,BSD-3-Clause @@ -55,7 +53,6 @@ github.com/klauspost/compress/flate,https://github.com/klauspost/compress/blob/v github.com/klauspost/pgzip,https://github.com/klauspost/pgzip/blob/v1.2.5/LICENSE,MIT github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 github.com/lestrrat-go/strftime,https://github.com/lestrrat-go/strftime/blob/v1.0.4/LICENSE,MIT -github.com/magiconair/properties,https://github.com/magiconair/properties/blob/v1.8.5/LICENSE.md,BSD-2-Clause github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT github.com/matttproud/golang_protobuf_extensions/pbutil,https://github.com/matttproud/golang_protobuf_extensions/blob/v1.0.4/LICENSE,Apache-2.0 github.com/mitchellh/copystructure,https://github.com/mitchellh/copystructure/blob/v1.2.0/LICENSE,MIT @@ -68,10 +65,9 @@ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c github.com/oklog/ulid,https://github.com/oklog/ulid/blob/v1.3.1/LICENSE,Apache-2.0 github.com/oliveagle/jsonpath,https://github.com/oliveagle/jsonpath/blob/2e52cf6e6852/LICENSE,MIT github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 -github.com/pelletier/go-toml,https://github.com/pelletier/go-toml/blob/v1.9.5/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -79,12 +75,8 @@ github.com/prometheus/statsd_exporter/pkg/mapper,https://github.com/prometheus/s github.com/robfig/cron/v3,https://github.com/robfig/cron/blob/v3.0.1/LICENSE,MIT github.com/shopspring/decimal,https://github.com/shopspring/decimal/blob/v1.2.0/LICENSE,MIT github.com/sirupsen/logrus,https://github.com/sirupsen/logrus/blob/v1.9.0/LICENSE,MIT -github.com/spf13/afero,https://github.com/spf13/afero/blob/v1.9.2/LICENSE.txt,Apache-2.0 github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.4.1/LICENSE,MIT -github.com/spf13/jwalterweatherman,https://github.com/spf13/jwalterweatherman/blob/v1.1.0/LICENSE,MIT github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause -github.com/spf13/viper,https://github.com/spf13/viper/blob/v1.10.1/LICENSE,MIT -github.com/subosito/gotenv,https://github.com/subosito/gotenv/blob/v1.2.0/LICENSE,MIT github.com/tektoncd/pipeline/pkg,https://github.com/tektoncd/pipeline/blob/v0.47.3/LICENSE,Apache-2.0 github.com/valyala/bytebufferpool,https://github.com/valyala/bytebufferpool/blob/v1.0.0/LICENSE,MIT github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1.2.1/LICENSE,MIT @@ -93,13 +85,13 @@ go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0 go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 @@ -108,7 +100,6 @@ google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/dcfb40 google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.54.0/LICENSE,Apache-2.0 google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.30.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause -gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.66.3/LICENSE,Apache-2.0 gopkg.in/jcmturner/aescts.v1,https://github.com/jcmturner/aescts/blob/v1.0.1/LICENSE,Apache-2.0 gopkg.in/jcmturner/dnsutils.v1,https://github.com/jcmturner/dnsutils/blob/v1.0.1/LICENSE,Apache-2.0 gopkg.in/jcmturner/gokrb5.v5,https://github.com/jcmturner/gokrb5/blob/v5.3.0/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/swf.csv b/backend/third_party_licenses/swf.csv index 5e4d9b5b3a..a9a4872bc8 100644 --- a/backend/third_party_licenses/swf.csv +++ b/backend/third_party_licenses/swf.csv @@ -71,7 +71,7 @@ github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/ github.com/pelletier/go-toml,https://github.com/pelletier/go-toml/blob/v1.9.5/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -94,13 +94,13 @@ go.opencensus.io,https://github.com/census-instrumentation/opencensus-go/blob/v0 go.uber.org/atomic,https://github.com/uber-go/atomic/blob/v1.10.0/LICENSE.txt,MIT go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT -golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync/semaphore,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/tekton-driver.csv b/backend/third_party_licenses/tekton-driver.csv index ac30c1ad0b..1494eae610 100644 --- a/backend/third_party_licenses/tekton-driver.csv +++ b/backend/third_party_licenses/tekton-driver.csv @@ -48,7 +48,7 @@ github.com/kelseyhightower/envconfig,https://github.com/kelseyhightower/envconfi github.com/kubeflow/pipelines/api/v2alpha1/go,https://github.com/kubeflow/pipelines/blob/758c91f76784/api/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 github.com/kubeflow/pipelines/kubernetes_platform/go/kubernetesplatform,https://github.com/kubeflow/pipelines/blob/bd9f74e34de6/kubernetes_platform/LICENSE,Apache-2.0 -github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e78ed557ddcb/third_party/ml-metadata/LICENSE,Apache-2.0 +github.com/kubeflow/pipelines/third_party/ml-metadata/go/ml_metadata,https://github.com/kubeflow/pipelines/blob/e1f0c010f800/third_party/ml-metadata/LICENSE,Apache-2.0 github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.7/LICENSE,MIT github.com/matttproud/golang_protobuf_extensions/pbutil,https://github.com/matttproud/golang_protobuf_extensions/blob/v1.0.4/LICENSE,Apache-2.0 github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 @@ -57,7 +57,7 @@ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -72,11 +72,11 @@ go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT gocloud.dev,https://github.com/google/go-cloud/blob/v0.22.0/LICENSE,Apache-2.0 golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause golang.org/x/xerrors,https://cs.opensource.google/go/x/xerrors/+/04be3eba:LICENSE,BSD-3-Clause diff --git a/backend/third_party_licenses/tekton-exithandler-controller.csv b/backend/third_party_licenses/tekton-exithandler-controller.csv index 7e4e0fe1f1..b40642a96e 100644 --- a/backend/third_party_licenses/tekton-exithandler-controller.csv +++ b/backend/third_party_licenses/tekton-exithandler-controller.csv @@ -39,7 +39,7 @@ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -52,11 +52,11 @@ go.uber.org/automaxprocs,https://github.com/uber-go/automaxprocs/blob/v1.4.0/LIC go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/tekton-exithandler-webhook.csv b/backend/third_party_licenses/tekton-exithandler-webhook.csv index e13acc23ed..1ceb4a7ddd 100644 --- a/backend/third_party_licenses/tekton-exithandler-webhook.csv +++ b/backend/third_party_licenses/tekton-exithandler-webhook.csv @@ -40,7 +40,7 @@ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -53,11 +53,11 @@ go.uber.org/automaxprocs,https://github.com/uber-go/automaxprocs/blob/v1.4.0/LIC go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/tekton-kfptask-controller.csv b/backend/third_party_licenses/tekton-kfptask-controller.csv index 7e4e0fe1f1..b40642a96e 100644 --- a/backend/third_party_licenses/tekton-kfptask-controller.csv +++ b/backend/third_party_licenses/tekton-kfptask-controller.csv @@ -39,7 +39,7 @@ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -52,11 +52,11 @@ go.uber.org/automaxprocs,https://github.com/uber-go/automaxprocs/blob/v1.4.0/LIC go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/tekton-kfptask-webhook.csv b/backend/third_party_licenses/tekton-kfptask-webhook.csv index e13acc23ed..1ceb4a7ddd 100644 --- a/backend/third_party_licenses/tekton-kfptask-webhook.csv +++ b/backend/third_party_licenses/tekton-kfptask-webhook.csv @@ -40,7 +40,7 @@ github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c github.com/opencontainers/go-digest,https://github.com/opencontainers/go-digest/blob/v1.0.0/LICENSE,Apache-2.0 github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 -github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.37.0/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.8.0/LICENSE,Apache-2.0 @@ -53,11 +53,11 @@ go.uber.org/automaxprocs,https://github.com/uber-go/automaxprocs/blob/v1.4.0/LIC go.uber.org/multierr,https://github.com/uber-go/multierr/blob/v1.8.0/LICENSE.txt,MIT go.uber.org/zap,https://github.com/uber-go/zap/blob/v1.24.0/LICENSE.txt,MIT golang.org/x/exp/maps,https://cs.opensource.google/go/x/exp/+/24139beb:LICENSE,BSD-3-Clause -golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/sync,https://cs.opensource.google/go/x/sync/+/v0.1.0:LICENSE,BSD-3-Clause -golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD-3-Clause -golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 diff --git a/backend/third_party_licenses/viewer.csv b/backend/third_party_licenses/viewer.csv index 361bffd618..68aac7bf5f 100644 --- a/backend/third_party_licenses/viewer.csv +++ b/backend/third_party_licenses/viewer.csv @@ -27,6 +27,7 @@ github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bac github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.2/LICENSE,Apache-2.0 github.com/munnerz/goautoneg,https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE,BSD-3-Clause github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause +<<<<<<< HEAD github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.13.0/LICENSE,Apache-2.0 github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.3.0/LICENSE,Apache-2.0 github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.37.0/LICENSE,Apache-2.0 @@ -39,6 +40,20 @@ golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.7.0:LICENSE,BSD golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.7.0:LICENSE,BSD-3-Clause golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/v0.3.0:LICENSE,BSD-3-Clause +======= +github.com/prometheus/client_golang/prometheus,https://github.com/prometheus/client_golang/blob/v1.12.1/LICENSE,Apache-2.0 +github.com/prometheus/client_model/go,https://github.com/prometheus/client_model/blob/v0.4.0/LICENSE,Apache-2.0 +github.com/prometheus/common,https://github.com/prometheus/common/blob/v0.32.1/LICENSE,Apache-2.0 +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg,https://github.com/prometheus/common/blob/v0.32.1/internal/bitbucket.org/ww/goautoneg/README.txt,BSD-3-Clause +github.com/prometheus/procfs,https://github.com/prometheus/procfs/blob/v0.7.3/LICENSE,Apache-2.0 +github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause +golang.org/x/net,https://cs.opensource.google/go/x/net/+/v0.10.0:LICENSE,BSD-3-Clause +golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/d3ed0bb2:LICENSE,BSD-3-Clause +golang.org/x/sys/unix,https://cs.opensource.google/go/x/sys/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/term,https://cs.opensource.google/go/x/term/+/v0.8.0:LICENSE,BSD-3-Clause +golang.org/x/text,https://cs.opensource.google/go/x/text/+/v0.9.0:LICENSE,BSD-3-Clause +golang.org/x/time/rate,https://cs.opensource.google/go/x/time/+/90d013bb:LICENSE,BSD-3-Clause +>>>>>>> kfp/master gomodules.xyz/jsonpatch/v2,https://github.com/gomodules/jsonpatch/blob/v2.2.0/v2/LICENSE,Apache-2.0 google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.30.0/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause diff --git a/backend/update_requirements.sh b/backend/update_requirements.sh new file mode 100755 index 0000000000..920940e909 --- /dev/null +++ b/backend/update_requirements.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# This image should be in sync with Dockerfile. +IMAGE="python:3.7" +../hack/update-requirements.sh $IMAGE requirements.txt diff --git a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_hosting.py b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_hosting.py index 65fd738e48..317a841537 100644 --- a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_hosting.py +++ b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_hosting.py @@ -28,7 +28,6 @@ def test_create_v2_endpoint(kfp_client, experiment_id, boto3_session, test_file_ shallow_canary=True, ) ) - k8s_client = ack_utils.k8s_client() input_model_name = utils.generate_random_string(10) + "-v2-model" input_endpoint_config_name = ( utils.generate_random_string(10) + "-v2-endpoint-config" @@ -63,7 +62,7 @@ def test_create_v2_endpoint(kfp_client, experiment_id, boto3_session, test_file_ ) endpoint_describe = ack_utils._get_resource( - k8s_client, input_endpoint_name, "endpoints" + input_endpoint_name, "endpoints" ) outputs = { @@ -142,11 +141,11 @@ def test_create_v2_endpoint(kfp_client, experiment_id, boto3_session, test_file_ ) utils.remove_dir(download_dir) finally: - ack_utils._delete_resource(k8s_client, input_endpoint_name, "endpoints") + ack_utils._delete_resource(input_endpoint_name, "endpoints") ack_utils._delete_resource( - k8s_client, input_endpoint_config_name, "endpointconfigs" + input_endpoint_config_name, "endpointconfigs" ) - ack_utils._delete_resource(k8s_client, input_model_name, "models") + ack_utils._delete_resource(input_model_name, "models") @pytest.mark.v2 @@ -159,7 +158,6 @@ def test_terminate_v2_endpoint(kfp_client, experiment_id): os.path.join(download_dir, "config.yaml"), ) ) - k8s_client = ack_utils.k8s_client() input_model_name = utils.generate_random_string(10) + "-v2-model" input_endpoint_config_name = ( utils.generate_random_string(10) + "-v2-endpoint-config" @@ -181,7 +179,6 @@ def test_terminate_v2_endpoint(kfp_client, experiment_id): "running", ) assert ack_utils.wait_for_condition( - k8s_client, input_endpoint_name, ack_utils.does_endpoint_exist, wait_periods=12, @@ -189,7 +186,6 @@ def test_terminate_v2_endpoint(kfp_client, experiment_id): ) kfp_client_utils.terminate_run(kfp_client, run_id) assert ack_utils.wait_for_condition( - k8s_client, input_endpoint_name, ack_utils.is_endpoint_deleted, wait_periods=20, @@ -197,6 +193,6 @@ def test_terminate_v2_endpoint(kfp_client, experiment_id): ) finally: ack_utils._delete_resource( - k8s_client, input_endpoint_config_name, "endpointconfigs" + input_endpoint_config_name, "endpointconfigs" ) - ack_utils._delete_resource(k8s_client, input_model_name, "models") + ack_utils._delete_resource(input_model_name, "models") diff --git a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_job_definition_component.py b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_job_definition_component.py index 2e8eed1f42..8a1dcf279a 100644 --- a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_job_definition_component.py +++ b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_job_definition_component.py @@ -33,7 +33,6 @@ def test_job_definitions(kfp_client, experiment_id, test_file_dir, deploy_endpoi os.path.join(download_dir, "config.yaml"), ) ) - k8s_client = ack_utils.k8s_client() job_definition_name = ( utils.generate_random_string(10) + "-v2-" + test_params["TestName"] ) @@ -56,7 +55,7 @@ def test_job_definitions(kfp_client, experiment_id, test_file_dir, deploy_endpoi # Verify if the job definition CR is created job_definition_describe = ack_utils._get_resource( - k8s_client, job_definition_name, test_params["Plural"] + job_definition_name, test_params["Plural"] ) assert ( job_definition_name @@ -88,5 +87,5 @@ def test_job_definitions(kfp_client, experiment_id, test_file_dir, deploy_endpoi finally: ack_utils._delete_resource( - k8s_client, job_definition_name, test_params["Plural"] + job_definition_name, test_params["Plural"] ) diff --git a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_schedule_component.py b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_schedule_component.py index 7edd7bd853..30f56663fd 100644 --- a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_schedule_component.py +++ b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_monitoring_schedule_component.py @@ -59,7 +59,6 @@ def test_create_v2_monitoring_schedule( os.path.join(download_dir, "config.yaml"), ) ) - k8s_client = ack_utils.k8s_client() # parameters for model bias job definition job_definition_name = ( @@ -92,7 +91,7 @@ def test_create_v2_monitoring_schedule( # Verify if the job definition CR is created properly job_definition_describe = ack_utils._get_resource( - k8s_client, job_definition_name, "modelbiasjobdefinitions" + job_definition_name, "modelbiasjobdefinitions" ) assert ( job_definition_name @@ -107,7 +106,7 @@ def test_create_v2_monitoring_schedule( # Verify if monitoring schedule CR is created properly monitoring_schedule_describe = ack_utils._get_resource( - k8s_client, monitoring_schedule_name, "monitoringschedules" + monitoring_schedule_name, "monitoringschedules" ) assert ( monitoring_schedule_name @@ -124,14 +123,12 @@ def test_create_v2_monitoring_schedule( finally: ack_utils._delete_resource( - k8s_client, job_definition_name, "modelbiasjobdefinitions", wait_periods=10, period_length=30, ) ack_utils._delete_resource( - k8s_client, monitoring_schedule_name, "monitoringschedules", wait_periods=10, @@ -164,7 +161,6 @@ def test_update_v2_monitoring_schedule( os.path.join(download_dir, "config.yaml"), ) ) - k8s_client = ack_utils.k8s_client() # parameters for job definition test_params["Arguments"][test_params["JobInputName"]]["endpointInput"][ @@ -202,7 +198,7 @@ def test_update_v2_monitoring_schedule( # Verify if monitoring schedule CR is created properly monitoring_schedule_describe = ack_utils._get_resource( - k8s_client, monitoring_schedule_name, "monitoringschedules" + monitoring_schedule_name, "monitoringschedules" ) assert ( monitoring_schedule_name @@ -221,7 +217,7 @@ def test_update_v2_monitoring_schedule( # Verify if the job definition CR is created properly job_definition_1_describe = ack_utils._get_resource( - k8s_client, job_definition_name_1, "dataqualityjobdefinitions" + job_definition_name_1, "dataqualityjobdefinitions" ) assert ( job_definition_name_1 @@ -262,7 +258,7 @@ def test_update_v2_monitoring_schedule( # Verify if monitoring schedule is updated with correct job definition monitoring_schedule_updated_describe = ack_utils._get_resource( - k8s_client, monitoring_schedule_name, "monitoringschedules" + monitoring_schedule_name, "monitoringschedules" ) assert ( monitoring_schedule_updated_describe["status"]["monitoringScheduleStatus"] @@ -277,7 +273,7 @@ def test_update_v2_monitoring_schedule( # Verify if the new job definition CR is created properly job_definition_2_describe = ack_utils._get_resource( - k8s_client, job_definition_name_2, "dataqualityjobdefinitions" + job_definition_name_2, "dataqualityjobdefinitions" ) assert ( job_definition_name_2 @@ -296,21 +292,18 @@ def test_update_v2_monitoring_schedule( finally: ack_utils._delete_resource( - k8s_client, job_definition_name_1, test_params["Plural"], wait_periods=10, period_length=30, ) ack_utils._delete_resource( - k8s_client, job_definition_name_2, test_params["Plural"], wait_periods=10, period_length=30, ) ack_utils._delete_resource( - k8s_client, monitoring_schedule_name, "monitoringschedules", wait_periods=10, diff --git a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_train_component.py b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_train_component.py index 34e1dc7e62..19378beafd 100644 --- a/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_train_component.py +++ b/components/aws/sagemaker/tests/integration_tests/component_tests/test_v2_train_component.py @@ -18,7 +18,6 @@ ], ) def test_trainingjobV2(kfp_client, experiment_id, test_file_dir): - k8s_client = ack_utils.k8s_client() test_file_dir = "resources/config/ack-training-job" download_dir = utils.mkdir(os.path.join(test_file_dir + "/generated")) test_params = utils.load_params( @@ -68,7 +67,7 @@ def test_trainingjobV2(kfp_client, experiment_id, test_file_dir): # Verify Training job was successful on SageMaker print(f"training job name: {input_job_name}") - train_response = ack_utils._get_resource(k8s_client, input_job_name, "trainingjobs") + train_response = ack_utils._get_resource(input_job_name, "trainingjobs") assert ( train_response["status"]["trainingJobStatus"] == output_training_job_status @@ -87,7 +86,6 @@ def test_trainingjobV2(kfp_client, experiment_id, test_file_dir): @pytest.mark.v2 def test_terminate_trainingjob(kfp_client, experiment_id): - k8s_client = ack_utils.k8s_client() test_file_dir = "resources/config/ack-training-job" download_dir = utils.mkdir(os.path.join(test_file_dir + "/generated_terminate")) @@ -114,7 +112,7 @@ def test_terminate_trainingjob(kfp_client, experiment_id): kfp_client_utils.terminate_run(kfp_client, run_id) desiredStatuses = ["Stopping", "Stopped"] training_status_reached = ack_utils.wait_for_trainingjob_status( - k8s_client, input_job_name, desiredStatuses, 10, 6 + input_job_name, desiredStatuses, 10, 6 ) assert training_status_reached diff --git a/components/aws/sagemaker/tests/integration_tests/utils/ack_utils.py b/components/aws/sagemaker/tests/integration_tests/utils/ack_utils.py index e5751c3672..ae48426dcf 100644 --- a/components/aws/sagemaker/tests/integration_tests/utils/ack_utils.py +++ b/components/aws/sagemaker/tests/integration_tests/utils/ack_utils.py @@ -7,13 +7,14 @@ def k8s_client(): return config.new_client_from_config() -def _get_resource(k8s_client, job_name, plural): +def _get_resource(job_name, plural): """Get the custom resource detail similar to: kubectl describe JOB_NAME -n NAMESPACE. Returns: None or object: None if the resource doesn't exist in server or there is an error, otherwise the custom object. """ - _api = client.CustomObjectsApi(k8s_client) + # Instantiate a new client every time to avoid connection issues. + _api = client.CustomObjectsApi(k8s_client()) namespace = os.environ.get("NAMESPACE") try: job_description = _api.get_namespaced_custom_object( @@ -29,12 +30,12 @@ def _get_resource(k8s_client, job_name, plural): return job_description -def _delete_resource(k8s_client, job_name, plural, wait_periods=10, period_length=20): +def _delete_resource(job_name, plural, wait_periods=10, period_length=20): """Delete the custom resource Returns: True or False: True if the resource is deleted, False if the resource deletion times out """ - _api = client.CustomObjectsApi(k8s_client) + _api = client.CustomObjectsApi(k8s_client()) namespace = os.environ.get("NAMESPACE") try: @@ -50,7 +51,7 @@ def _delete_resource(k8s_client, job_name, plural, wait_periods=10, period_lengt for _ in range(wait_periods): sleep(period_length) - if _get_resource(k8s_client, job_name, plural) is None: + if _get_resource(job_name, plural) is None: print(f"Resource {job_name} deleted successfully.") return True @@ -60,10 +61,10 @@ def _delete_resource(k8s_client, job_name, plural, wait_periods=10, period_lengt # TODO: Make this a generalized function for non-job resources. def wait_for_trainingjob_status( - k8s_client, training_job_name, desiredStatuses, wait_periods, period_length + training_job_name, desiredStatuses, wait_periods, period_length ): for _ in range(wait_periods): - response = _get_resource(k8s_client, training_job_name, "trainingjobs") + response = _get_resource(training_job_name, "trainingjobs") if response["status"]["trainingJobStatus"] in desiredStatuses: return True sleep(period_length) @@ -71,19 +72,19 @@ def wait_for_trainingjob_status( def wait_for_condition( - k8s_client, resource_name, validator_function, wait_periods=10, period_length=8 + resource_name, validator_function, wait_periods=10, period_length=8 ): for _ in range(wait_periods): - if not validator_function(k8s_client, resource_name): + if not validator_function(resource_name): sleep(period_length) else: return True return False -def does_endpoint_exist(k8s_client, endpoint_name): +def does_endpoint_exist(endpoint_name): try: - response = _get_resource(k8s_client, endpoint_name, "endpoints") + response = _get_resource(endpoint_name, "endpoints") if response: return True if response is None: # kubernetes module error @@ -92,8 +93,8 @@ def does_endpoint_exist(k8s_client, endpoint_name): return False -def is_endpoint_deleted(k8s_client, endpoint_name): - response = _get_resource(k8s_client, endpoint_name, "endpoints") +def is_endpoint_deleted(endpoint_name): + response = _get_resource(endpoint_name, "endpoints") if response: return False if response is None: diff --git a/contrib/components/openvino/model_convert/README.md b/components/contrib/openvino/model_convert/README.md similarity index 100% rename from contrib/components/openvino/model_convert/README.md rename to components/contrib/openvino/model_convert/README.md diff --git a/contrib/components/openvino/model_convert/containers/Dockerfile b/components/contrib/openvino/model_convert/containers/Dockerfile similarity index 100% rename from contrib/components/openvino/model_convert/containers/Dockerfile rename to components/contrib/openvino/model_convert/containers/Dockerfile diff --git a/contrib/components/openvino/model_convert/containers/convert_model.py b/components/contrib/openvino/model_convert/containers/convert_model.py similarity index 100% rename from contrib/components/openvino/model_convert/containers/convert_model.py rename to components/contrib/openvino/model_convert/containers/convert_model.py diff --git a/contrib/components/openvino/ovms-deployer/README.md b/components/contrib/openvino/ovms-deployer/README.md similarity index 100% rename from contrib/components/openvino/ovms-deployer/README.md rename to components/contrib/openvino/ovms-deployer/README.md diff --git a/contrib/components/openvino/ovms-deployer/containers/Dockerfile b/components/contrib/openvino/ovms-deployer/containers/Dockerfile similarity index 100% rename from contrib/components/openvino/ovms-deployer/containers/Dockerfile rename to components/contrib/openvino/ovms-deployer/containers/Dockerfile diff --git a/contrib/components/openvino/ovms-deployer/containers/apply_template.py b/components/contrib/openvino/ovms-deployer/containers/apply_template.py similarity index 100% rename from contrib/components/openvino/ovms-deployer/containers/apply_template.py rename to components/contrib/openvino/ovms-deployer/containers/apply_template.py diff --git a/contrib/components/openvino/ovms-deployer/containers/classes.py b/components/contrib/openvino/ovms-deployer/containers/classes.py similarity index 100% rename from contrib/components/openvino/ovms-deployer/containers/classes.py rename to components/contrib/openvino/ovms-deployer/containers/classes.py diff --git a/contrib/components/openvino/ovms-deployer/containers/deploy.sh b/components/contrib/openvino/ovms-deployer/containers/deploy.sh similarity index 100% rename from contrib/components/openvino/ovms-deployer/containers/deploy.sh rename to components/contrib/openvino/ovms-deployer/containers/deploy.sh diff --git a/contrib/components/openvino/ovms-deployer/containers/evaluate.py b/components/contrib/openvino/ovms-deployer/containers/evaluate.py similarity index 100% rename from contrib/components/openvino/ovms-deployer/containers/evaluate.py rename to components/contrib/openvino/ovms-deployer/containers/evaluate.py diff --git a/contrib/components/openvino/ovms-deployer/containers/ovms.j2 b/components/contrib/openvino/ovms-deployer/containers/ovms.j2 similarity index 100% rename from contrib/components/openvino/ovms-deployer/containers/ovms.j2 rename to components/contrib/openvino/ovms-deployer/containers/ovms.j2 diff --git a/contrib/components/openvino/ovms-deployer/containers/requirements.txt b/components/contrib/openvino/ovms-deployer/containers/requirements.txt similarity index 100% rename from contrib/components/openvino/ovms-deployer/containers/requirements.txt rename to components/contrib/openvino/ovms-deployer/containers/requirements.txt diff --git a/contrib/components/openvino/predict/README.md b/components/contrib/openvino/predict/README.md similarity index 100% rename from contrib/components/openvino/predict/README.md rename to components/contrib/openvino/predict/README.md diff --git a/contrib/components/openvino/predict/containers/Dockerfile b/components/contrib/openvino/predict/containers/Dockerfile similarity index 100% rename from contrib/components/openvino/predict/containers/Dockerfile rename to components/contrib/openvino/predict/containers/Dockerfile diff --git a/contrib/components/openvino/predict/containers/classes.py b/components/contrib/openvino/predict/containers/classes.py similarity index 100% rename from contrib/components/openvino/predict/containers/classes.py rename to components/contrib/openvino/predict/containers/classes.py diff --git a/contrib/components/openvino/predict/containers/predict.py b/components/contrib/openvino/predict/containers/predict.py similarity index 100% rename from contrib/components/openvino/predict/containers/predict.py rename to components/contrib/openvino/predict/containers/predict.py diff --git a/contrib/components/openvino/predict/containers/requirements.txt b/components/contrib/openvino/predict/containers/requirements.txt similarity index 100% rename from contrib/components/openvino/predict/containers/requirements.txt rename to components/contrib/openvino/predict/containers/requirements.txt diff --git a/contrib/components/openvino/tf-slim/README.md b/components/contrib/openvino/tf-slim/README.md similarity index 100% rename from contrib/components/openvino/tf-slim/README.md rename to components/contrib/openvino/tf-slim/README.md diff --git a/contrib/components/openvino/tf-slim/containers/Dockerfile b/components/contrib/openvino/tf-slim/containers/Dockerfile similarity index 100% rename from contrib/components/openvino/tf-slim/containers/Dockerfile rename to components/contrib/openvino/tf-slim/containers/Dockerfile diff --git a/contrib/components/openvino/tf-slim/containers/slim_model.py b/components/contrib/openvino/tf-slim/containers/slim_model.py similarity index 100% rename from contrib/components/openvino/tf-slim/containers/slim_model.py rename to components/contrib/openvino/tf-slim/containers/slim_model.py diff --git a/components/google-cloud/google_cloud_pipeline_components/container/Dockerfile b/components/google-cloud/Dockerfile similarity index 66% rename from components/google-cloud/google_cloud_pipeline_components/container/Dockerfile rename to components/google-cloud/Dockerfile index 015614e3f8..c4fba596e8 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/Dockerfile +++ b/components/google-cloud/Dockerfile @@ -28,8 +28,14 @@ RUN pip3 install -U google-cloud-storage RUN pip3 install -U google-api-python-client # Required by dataflow_launcher -# b/238481913: Pinning the version of apache_beam to below 2.34 for now -RUN pip3 install -U "apache_beam[gcp]<2.34.0" +# Pin to `2.50.0` for compatibility with `google-cloud-aiplatform`, which +# depends on `shapely<3.0.0dev`. +# Prefer an exact pin, since GCPC's apache_beam version must match the +# version the in custom Dataflow worker images for the Dataflow job to succeed. +# Inexact pins risk that the apache_beam in GCPC drifts away from a +# user-specified version in the image. +# From docs: """When running your pipeline, launch the pipeline using the Apache Beam SDK with the same version and language version as the SDK on your custom container image. This step avoids unexpected errors from incompatible dependencies or SDKs.""" https://cloud.google.com/dataflow/docs/guides/using-custom-containers#before_you_begin_2 +RUN pip3 install -U "apache_beam[gcp]==2.50.0" # Required for sklearn/train_test_split_jsonl RUN pip3 install -U "fsspec>=0.7.4" "gcsfs>=0.6.0" "pandas<=1.3.5" "scikit-learn<=1.0.2" @@ -38,7 +44,7 @@ RUN pip3 install -U "fsspec>=0.7.4" "gcsfs>=0.6.0" "pandas<=1.3.5" "scikit-learn RUN pip3 install -U google-cloud-notebooks # Install main package -RUN pip3 install "git+https://github.com/kubeflow/pipelines.git@google-cloud-pipeline-components-2.0.0#egg=google-cloud-pipeline-components&subdirectory=components/google-cloud" +RUN pip3 install "git+https://github.com/kubeflow/pipelines.git@google-cloud-pipeline-components-2.5.0#egg=google-cloud-pipeline-components&subdirectory=components/google-cloud" # Note that components can override the container entry ponint. ENTRYPOINT ["python3","-m","google_cloud_pipeline_components.container.v1.aiplatform.remote_runner"] diff --git a/components/google-cloud/OWNERS b/components/google-cloud/OWNERS deleted file mode 100644 index 4d945bea26..0000000000 --- a/components/google-cloud/OWNERS +++ /dev/null @@ -1,11 +0,0 @@ -# Google Cloud components - MB SDK -approvers: - - chensun - - IronPan - - neuromage - - sasha-gitg - - sinachavoshi -reviewers: - - chensun - - sasha-gitg - - sinachavoshi diff --git a/components/google-cloud/README.md b/components/google-cloud/README.md index 58795e172a..ffe894c026 100644 --- a/components/google-cloud/README.md +++ b/components/google-cloud/README.md @@ -12,10 +12,9 @@ Please see the [Google Cloud Pipeline Components user guide](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction). ### API documentation -Please see the [Google Cloud Pipeline Components API reference documentation](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.41/). +Please see the [Google Cloud Pipeline Components API reference documentation](https://google-cloud-pipeline-components.readthedocs.io/). ### Release details - For details about previous and upcoming releases, please see the [release notes](https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/RELEASE.md). ## Examples diff --git a/components/google-cloud/RELEASE.md b/components/google-cloud/RELEASE.md index 450778b9fe..6fb3b383f0 100644 --- a/components/google-cloud/RELEASE.md +++ b/components/google-cloud/RELEASE.md @@ -1,5 +1,62 @@ ## Upcoming release +## Release 2.5.0 +* Upload tensorboard metrics from `preview.llm.rlhf_pipeline` if a `tensorboard_resource_id` is provided at runtime. +* Support `incremental_train_base_model`, `parent_model`, `is_default_version`, `model_version_aliases`, `model_version_description` in `AutoMLImageTrainingJobRunOp`. +* Add `preview.automl.vision` and `DataConverterJobOp`. +* Set display names for `preview.llm` pipelines. +* Add sliced evaluation metrics support for custom and unstructured AutoML models in evaluation pipeline and evaluation pipeline with feature attribution. +* Support `service_account` in `ModelBatchPredictOp`. +* Release `DataflowFlexTemplateJobOp` to GA namespace (`v1.dataflow.DataflowFlexTemplateJobOp`). +* Make `model_checkpoint` optional for `preview.llm.infer_pipeline`. If not provided, the base model associated with the `large_model_reference` will be used. +* Bump `apache_beam[gcp]` version in GCPC container image from `<2.34.0` to `==2.50.0` for compatibility with `google-cloud-aiplatform`, which depends on `shapely<3.0.0dev`. Note: upgrades to `google-cloud-pipeline-components`>=2.5.0 and later may require using a Dataflow worker image with `apache_beam==2.50.0`. +* Apply latest GCPC image vulnerability resolutions (base OS and software updates) +* Add support for customizing model_parameters (maxOutputTokens, topK, topP, and + temperature) in LLM eval text generation and LLM eval text classification + pipelines. + +## Release 2.4.1 +* Disable caching for LLM pipeline tasks that store temporary artifacts. +* Fix the mismatched arguments in 2.4.0 for the Feature Transform Engine component. +* Apply latest GCPC image vulnerability resolutions (base OS and software updates). + +## Release 2.4.0 +* Add support for running tasks on a `PersistentResource` (see [CustomJobSpec](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/CustomJobSpec)) via `persistent_resource_id` parameter on `preview.custom_job.CustomTrainingJobOp` and `preview.custom_job.create_custom_training_job_from_component` +* Fix use of `encryption_spec_key_name` in `v1.custom_job.CustomTrainingJobOp` and `v1.custom_job.create_custom_training_job_from_component` +* Add feature_selection_pipeline to preview.automl.tabular. +* Bump supported KFP versions to kfp>=2.0.0b10,<=2.2.0 +* Add `time_series_dense_encoder_forecasting_pipeline`, `learn_to_learn_forecasting_pipeline`, `sequence_to_sequence_forecasting_pipeline`, and `temporal_fusion_transformer_forecasting_pipeline` to `preview.automl.forecasting`. +* Add support for customizing evaluation display name on `v1` and `preview` `model_evaluation` pipelines. +* Include model version ID in `v1.model.upload_model.ModelUploadOp`'s `VertexModel` output (key: `model`). The URI and metadata `resourceName` field in the outputted `VertexModel` now have `@` appended, corresponding to the model that was just created. Downstream components `DeleteModel` and `UndeployModel` will respect the model version if provided. +* Bump KFP SDK upper bound to 2.3.0 +* Apply latest GCPC image vulnerability resolutions (base OS and software updates) + +## Release 2.3.1 +* Make LLM pipelines compatible with KFP SDK 2.1.3 +* Require KFP SDK <=2.1.3 +* Apply latest GCPC image vulnerability resolutions (base OS and software updates) + +## Release 2.3.0 +* Add `preview.llm.infer_pipeline` and `preview.llm.rlhf_pipeline` +* Add `automl_tabular_tabnet_trainer` and `automl_tabular_wide_and_deep_trainer` to `preview.automl.tabular` and `v1.automl.tabular` +* Minor feature additions to AutoML components +* Apply latest GCPC image vulnerability resolutions (base OS and software updates) + +## Release 2.2.0 +* Add `preview.model_evaluation.evaluation_llm_classification_pipeline.evaluation_llm_classification_pipeline` +* Change AutoML Vision Error Analysis pipeline names (`v1.model_evaluation.vision_model_error_analysis_pipeline' and 'v1.model_evaluation.evaluated_annotation_pipeline') +* Apply latest GCPC image vulnerability resolutions (base OS and software updates) + +## Release 2.1.1 +* Add `preview.model_evaluation.FeatureAttributionGraphComponentOp` pipeline +* Apply latest GCPC image vulnerability resolutions (base OS and software updates) + +## Release 2.1.0 +* Add AutoML tabular and forecasting components to `preview` namespace +* Fix bug where `parent_model` parameter of `ModelUploadOp` ignored +* Fix circular import bug for model evaluation components +* Apply latest GCPC image vulnerability resolutions (base OS and software updates) + ## Release 2.0.0 Google Cloud Pipeline Components v2 is generally available! diff --git a/components/google-cloud/google_cloud_pipeline_components/container/cloudbuild.yaml b/components/google-cloud/cloudbuild.yaml similarity index 55% rename from components/google-cloud/google_cloud_pipeline_components/container/cloudbuild.yaml rename to components/google-cloud/cloudbuild.yaml index f4825cc5bf..0b9ec701bd 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/cloudbuild.yaml +++ b/components/google-cloud/cloudbuild.yaml @@ -1,5 +1,6 @@ steps: - name: 'gcr.io/kaniko-project/executor:latest' args: - - --destination=gcr.io/$PROJECT_ID/google-cloud-pipeline-components:2.0.0 + - --destination=gcr.io/$PROJECT_ID/google-cloud-pipeline-components:$_IMAGE_SUFFIX + - --destination=gcr.io/$PROJECT_ID/google-cloud-pipeline-components:public-image-gcpc-v2-scan - --cache=false diff --git a/components/google-cloud/docs/.readthedocs.yml b/components/google-cloud/docs/.readthedocs.yml index 2febfec5c1..dc99a9d93a 100644 --- a/components/google-cloud/docs/.readthedocs.yml +++ b/components/google-cloud/docs/.readthedocs.yml @@ -3,9 +3,12 @@ version: 2 sphinx: configuration: components/google-cloud/docs/source/conf.py python: - version: 3.7 install: - method: pip path: components/google-cloud extra_requirements: - docs +build: + os: ubuntu-22.04 + tools: + python: "3.7" \ No newline at end of file diff --git a/components/google-cloud/docs/Makefile b/components/google-cloud/docs/Makefile deleted file mode 100644 index d0c3cbf102..0000000000 --- a/components/google-cloud/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/components/google-cloud/docs/make.bat b/components/google-cloud/docs/make.bat deleted file mode 100644 index 6247f7e231..0000000000 --- a/components/google-cloud/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/components/google-cloud/docs/source/api/preview/automl/forecasting.rst b/components/google-cloud/docs/source/api/preview/automl/forecasting.rst new file mode 100644 index 0000000000..761f8d5180 --- /dev/null +++ b/components/google-cloud/docs/source/api/preview/automl/forecasting.rst @@ -0,0 +1,4 @@ +AutoML Forecasting +========================== + +.. automodule:: preview.automl.forecasting \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/preview/automl/index.rst b/components/google-cloud/docs/source/api/preview/automl/index.rst new file mode 100644 index 0000000000..1aedabd617 --- /dev/null +++ b/components/google-cloud/docs/source/api/preview/automl/index.rst @@ -0,0 +1,8 @@ +AutoML +============= + +.. toctree:: + :maxdepth: 1 + + forecasting + tabular \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/preview/automl/tabular.rst b/components/google-cloud/docs/source/api/preview/automl/tabular.rst new file mode 100644 index 0000000000..4b9d913ad2 --- /dev/null +++ b/components/google-cloud/docs/source/api/preview/automl/tabular.rst @@ -0,0 +1,4 @@ +AutoML Tabular +========================== + +.. automodule:: preview.automl.tabular \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/preview/custom_job.rst b/components/google-cloud/docs/source/api/preview/custom_job.rst new file mode 100644 index 0000000000..ed3a195457 --- /dev/null +++ b/components/google-cloud/docs/source/api/preview/custom_job.rst @@ -0,0 +1,4 @@ +Custom Job +========================== + +.. automodule:: preview.custom_job \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/preview/index.rst b/components/google-cloud/docs/source/api/preview/index.rst index 20cdbf132b..8074a6c6b8 100644 --- a/components/google-cloud/docs/source/api/preview/index.rst +++ b/components/google-cloud/docs/source/api/preview/index.rst @@ -4,5 +4,8 @@ Preview Components .. toctree:: :maxdepth: 1 + automl/index + custom_job dataflow + llm model_evaluation \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/preview/llm.rst b/components/google-cloud/docs/source/api/preview/llm.rst new file mode 100644 index 0000000000..e68c374216 --- /dev/null +++ b/components/google-cloud/docs/source/api/preview/llm.rst @@ -0,0 +1,4 @@ +LLM +========================== + +.. automodule:: preview.llm \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/v1/automl/forecasting.rst b/components/google-cloud/docs/source/api/v1/automl/forecasting.rst new file mode 100644 index 0000000000..de710136ca --- /dev/null +++ b/components/google-cloud/docs/source/api/v1/automl/forecasting.rst @@ -0,0 +1,4 @@ +AutoML Forecasting +========================== + +.. automodule:: v1.automl.forecasting \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/v1/automl/index.rst b/components/google-cloud/docs/source/api/v1/automl/index.rst index 92f20b20af..4e9c83f04a 100644 --- a/components/google-cloud/docs/source/api/v1/automl/index.rst +++ b/components/google-cloud/docs/source/api/v1/automl/index.rst @@ -4,4 +4,6 @@ AutoML .. toctree:: :maxdepth: 1 + forecasting + tabular training_job \ No newline at end of file diff --git a/components/google-cloud/docs/source/api/v1/automl/tabular.rst b/components/google-cloud/docs/source/api/v1/automl/tabular.rst new file mode 100644 index 0000000000..dabbebf2e9 --- /dev/null +++ b/components/google-cloud/docs/source/api/v1/automl/tabular.rst @@ -0,0 +1,4 @@ +AutoML Tabular +========================== + +.. automodule:: v1.automl.tabular \ No newline at end of file diff --git a/components/google-cloud/docs/source/conf.py b/components/google-cloud/docs/source/conf.py index a169f2ee86..a12613d28f 100644 --- a/components/google-cloud/docs/source/conf.py +++ b/components/google-cloud/docs/source/conf.py @@ -19,6 +19,7 @@ import textwrap from typing import List +import commonmark import docstring_parser from google_cloud_pipeline_components import utils from kfp import components @@ -112,22 +113,6 @@ def __getitem__(self, type_) -> str: dsl.Output = Output -# order from earliest to latest -# start with 2.0.0b3, which is the first time we're using the new theme -V2_DROPDOWN_VERSIONS = [ - '2.0.0b3', - '2.0.0b4', - '2.0.0b5', - '2.0.0', -] - -# The short X.Y version -# update for each release -LATEST_VERSION = V2_DROPDOWN_VERSIONS[-1] - -# The full version, including alpha/beta/rc tags -release = LATEST_VERSION - # -- General configuration --------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. @@ -208,14 +193,7 @@ def __getitem__(self, type_) -> str: }], 'font': {'text': 'Open Sans'}, 'version_dropdown': True, - 'version_info': [ - { - 'version': f'https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-{version}', - 'title': version, - 'aliases': [], - } - for version in reversed(V2_DROPDOWN_VERSIONS) - ], + 'version_json': 'https://raw.githubusercontent.com/kubeflow/pipelines/master/components/google-cloud/docs/source/versions.json', # "toc_title_is_page_title": True, } # Add any paths that contain templates here, relative to this directory. @@ -350,6 +328,7 @@ def remove_after_returns_in_place(lines: List[str]) -> bool: return False def process_named_docstring_returns(app, what, name, obj, options, lines): + markdown_to_rst(lines) if getattr(obj, '_is_component', False): has_returns_section = remove_after_returns_in_place(lines) if has_returns_section: @@ -357,6 +336,15 @@ def process_named_docstring_returns(app, what, name, obj, options, lines): lines.extend([':returns:', '']) lines.extend(returns_section) + +def markdown_to_rst(lines: List[str]) -> List[str]: + md = '\n'.join(lines) + ast = commonmark.Parser().parse(md) + rst = commonmark.ReStructuredTextRenderer().render(ast) + lines.clear() + lines += rst.splitlines() + + def setup(app): app.connect('autodoc-process-docstring', process_named_docstring_returns) app.connect( diff --git a/components/google-cloud/docs/source/versions.json b/components/google-cloud/docs/source/versions.json new file mode 100644 index 0000000000..3be6717cf2 --- /dev/null +++ b/components/google-cloud/docs/source/versions.json @@ -0,0 +1,62 @@ +[ + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.5.0", + "title": "2.5.0", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.4.1", + "title": "2.4.1", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.4.0", + "title": "2.4.0", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.3.1", + "title": "2.3.1", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.3.0", + "title": "2.3.0", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.2.0", + "title": "2.2.0", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.1.1", + "title": "2.1.1", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.1.0", + "title": "2.1.0", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.0.0", + "title": "2.0.0", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.0.0b5", + "title": "2.0.0b5", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.0.0b4", + "title": "2.0.0b4", + "aliases": [] + }, + { + "version": "https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-2.0.0b3", + "title": "2.0.0b3", + "aliases": [] + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components.gwsq b/components/google-cloud/google_cloud_pipeline_components.gwsq deleted file mode 100644 index b221919d16..0000000000 --- a/components/google-cloud/google_cloud_pipeline_components.gwsq +++ /dev/null @@ -1,14 +0,0 @@ -// corresponds to g/google-cloud-pipeline-components-approvers -send_cls_to('google-cloud-pipeline-components-approvers'); - -define Main { - // corresponds to ganpati google-cloud-pipeline-components-approvers.prod - list RequiredGcpcApprovers = mdb('google-cloud-pipeline-components-approvers'); - - // WANT_LGTM from a GCPC approver if the author is not a GCPC approver, a GCPC approver is not already assigned, and the author has begun assigning reviewers to their CL (not a WIP anymore) - if (!author_in(RequiredGcpcApprovers) && len(current_reviewers()) > 0) { - string SelectedApprover = list_to_string(select_from(RequiredGcpcApprovers)); - set_tag('WANT_LGTM', SelectedApprover, append=True); - } - -} \ No newline at end of file diff --git a/components/google-cloud/google_cloud_pipeline_components/__init__.py b/components/google-cloud/google_cloud_pipeline_components/__init__.py index 8489662cf6..42ec791d4b 100644 --- a/components/google-cloud/google_cloud_pipeline_components/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/__init__.py @@ -12,4 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. """Google Cloud Pipeline Components.""" -from google_cloud_pipeline_components.version import __version__ +import sys +import warnings + +if sys.version_info < (3, 8): + warnings.warn( + ( + 'Python 3.7 has reached end-of-life. Google Cloud Pipeline Components' + ' will drop support for Python 3.7 on April 23, 2024. To use new' + ' versions of the KFP SDK after that date, you will need to upgrade' + ' to Python >= 3.8. See https://devguide.python.org/versions/ for' + ' more details.' + ), + FutureWarning, + stacklevel=2, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_image.py b/components/google-cloud/google_cloud_pipeline_components/_image.py index 63dbc85814..2e8777527f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_image.py +++ b/components/google-cloud/google_cloud_pipeline_components/_image.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Constants for the GCPC image.""" from google_cloud_pipeline_components import version GCPC_IMAGE_NAME = 'gcr.io/ml-pipeline/google-cloud-pipeline-components' diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/__init__.py new file mode 100644 index 0000000000..aa8704bef8 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/bulk_inferrer.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/bulk_inferrer.py new file mode 100644 index 0000000000..0bb327fbf3 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/bulk_inferrer.py @@ -0,0 +1,87 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Container component that performs bulk inference.""" + +from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.llm import utils +import kfp + + +@kfp.dsl.container_component +def BulkInferrer( # pylint: disable=invalid-name + project: str, + location: str, + inputs_sequence_length: int, + targets_sequence_length: int, + accelerator_type: str, + accelerator_count: int, + machine_type: str, + image_uri: str, + dataset_split: str, + large_model_reference: str, + input_model: str, + input_dataset_path: str, + output_prediction: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + output_prediction_gcs_path: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + gcp_resources: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + sampling_strategy: str = 'greedy', +) -> kfp.dsl.ContainerSpec: # pylint: disable=g-doc-args + """Performs bulk inference. + + Args: + project: Project used to run the job. + location: Location used to run the job. + inputs_sequence_length: Maximum encoder/prefix length. Inputs will be padded + or truncated to this length. + targets_sequence_length: Maximum decoder steps. Outputs will be at most this + length. + accelerator_type: Type of accelerator. + accelerator_count: Number of accelerators. + machine_type: Type of machine. + image: Location of reward model Docker image. + input_model: Model to use for inference. + large_model_reference: Predefined model used to create the ``input_model``. + input_dataset_path: Path to dataset to use for inference. + sampling_strategy: The sampling strategy for inference. + dataset_split: Perform inference on this split of the input dataset. + + Returns: + output_prediction: Where to save the output prediction. + gcp_resources: GCP resources that can be used to track the custom finetuning + job. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_payload( + display_name='BulkInferrer', + accelerator_type=accelerator_type, + accelerator_count=accelerator_count, + machine_type=machine_type, + image_uri=image_uri, + args=[ + f'--input_model={input_model}', + f'--input_dataset={input_dataset_path}', + f'--dataset_split={dataset_split}', + f'--large_model_reference={large_model_reference}', + f'--inputs_sequence_length={inputs_sequence_length}', + f'--targets_sequence_length={targets_sequence_length}', + f'--sampling_strategy={sampling_strategy}', + f'--output_prediction={output_prediction}', + f'--output_prediction_gcs_path={output_prediction_gcs_path}', + ], + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/deploy_llm_model.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/deploy_llm_model.py new file mode 100644 index 0000000000..7fbad47ee3 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/deploy_llm_model.py @@ -0,0 +1,145 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Component for deploy_llm_model.""" + +from google_cloud_pipeline_components import _image +from kfp import dsl + + +# pylint: disable=g-import-not-at-top, invalid-name +# pylint: disable=g-doc-args +# pytype: disable=invalid-annotation +# pytype: disable=import-error +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def create_endpoint_and_deploy_model( + project: str, + location: str, + model_resource_name: str, + display_name: str, + regional_endpoint: str, + endpoint_resource_name: dsl.OutputPath(str), + create_endpoint_gcp_resources: dsl.OutputPath(str), + deploy_model_gcp_resources: dsl.OutputPath(str), + encryption_spec_key_name: str = '', + service_account: str = '', + deploy_model: bool = True, +): + """Creates a vertex endpoint and deploy the specified model. + + Args: + project: Name of the GCP project. + location: Location for model upload and deployment. + model_resource_name: Path to the created Model on Model Registry. + display_name: Name of the model (shown in Model Registry). + regional_endpoint: Regional API endpoint. + encryption_spec_key_name: Customer-managed encryption key. + service_account: If set, then a custom service account will be used. + deploy_model: Whether to deploy the model to an endpoint. Default is + ``True``. If ``False``, the model will not be deployed and output + artifacts will contain empty strings. + + Returns: + endpoint_resource_name: Path to the created endpoint on Online Prediction. + create_endpoint_gcp_resources: Serialized JSON of GCP resources for + creating an endpoint. + deploy_model_gcp_resources: Serialized JSON of GCP resources for deploying + the model. + """ + import json + import logging + import os + import sys + from typing import Any, Dict + + try: + from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner + except ImportError: + from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner + + def run_lro_remote_runner( + url: str, payload: Dict[str, Any], gcp_resources: str + ) -> Any: + remote_runner = lro_remote_runner.LroRemoteRunner(location) + lro = remote_runner.create_lro(url, json.dumps(payload), gcp_resources) + return remote_runner.poll_lro(lro=lro) + + try: + os.makedirs(os.path.dirname(endpoint_resource_name), exist_ok=True) + + if not deploy_model: + with open(endpoint_resource_name, 'w') as fout: + fout.write('') + return + + regional_endpoint = regional_endpoint.rstrip('/') + + create_endpoint_payload = { + 'displayName': display_name, + } + + pipeline_labels_str = os.getenv('VERTEX_AI_PIPELINES_RUN_LABELS') + if pipeline_labels_str: + create_endpoint_payload['labels'] = json.loads(pipeline_labels_str) + + if encryption_spec_key_name: + create_endpoint_payload['encryption_spec'] = { + 'kms_key_name': encryption_spec_key_name + } + + create_endpoint_lro = run_lro_remote_runner( + url=( + f'{regional_endpoint}/projects/{project}/locations/{location}' + '/endpoints' + ), + payload=create_endpoint_payload, + gcp_resources=create_endpoint_gcp_resources, + ) + + response_endpoint = create_endpoint_lro['response']['name'] + with open(endpoint_resource_name, 'w') as fout: + fout.write(response_endpoint) + + logging.info( + 'Endpoint created successfully. Deploying model %s to endpoint', + model_resource_name, + ) + + deploy_model_payload = { + 'deployedModel': { + 'model': model_resource_name, + 'displayName': display_name, + 'automaticResources': {'minReplicaCount': 1, 'maxReplicaCount': 1}, + } + } + if service_account: + deploy_model_payload['deployedModel']['service_account'] = service_account + + _ = run_lro_remote_runner( + url=f'{regional_endpoint}/{response_endpoint}:deployModel', + payload=deploy_model_payload, + gcp_resources=deploy_model_gcp_resources, + ) + + logging.info('Model deployed successfully!') + except Exception as e: # pylint: disable=broad-exception-caught + if isinstance(e, ValueError): + raise + logging.exception(str(e)) + sys.exit(13) + + +# pytype: enable=import-error +# pytype: enable=invalid-annotation +# pylint: enable=g-doc-args +# pylint: enable=g-import-not-at-top, invalid-name diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py new file mode 100644 index 0000000000..af0c853c0a --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/env.py @@ -0,0 +1,40 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A collection of constants shared across components and pipelines.""" +import os + + +def get_private_image_tag() -> str: + return os.getenv('PRIVATE_IMAGE_TAG', '20231010_1107_RC00') + + +def get_use_test_machine_spec() -> bool: + str_value = os.getenv('USE_TEST_MACHINE_SPEC', 'False') + return str_value.lower() in {'true', '1'} + + +# Variables associated with private images: +CLOUD_ML_REGION = os.getenv('CLOUD_ML_REGION', 'europe-west4') +PRIVATE_ARTIFACT_REGISTRY_PROJECT: str = os.getenv( + 'PRIVATE_ARTIFACT_REGISTRY_PROJECT', 'vertex-ai-restricted' +) +PRIVATE_ARTIFACT_REGISTRY_LOCATION: str = os.getenv( + 'PRIVATE_ARTIFACT_REGISTRY_LOCATION', 'us' +) +PRIVATE_ARTIFACT_REGISTRY: str = os.getenv('PRIVATE_ARTIFACT_REGISTRY', 'rlhf') +PRIVATE_IMAGE_NAME_PREFIX: str = os.getenv('PRIVATE_IMAGE_NAME_PREFIX', 'rlhf_') +PRIVATE_IMAGE_TAG: str = get_private_image_tag() + +# Dataset variables: +TRAIN_SPLIT: str = 'train' diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/function_based.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/function_based.py new file mode 100644 index 0000000000..67b914811f --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/function_based.py @@ -0,0 +1,491 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python function-based components used in KFP pipelies.""" +import functools +from typing import List, NamedTuple, Optional + +from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components._implementation.llm import env +from kfp import dsl + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_machine_spec( + location: str, + use_test_spec: bool = False, +) -> NamedTuple( + 'MachineSpec', machine_type=str, accelerator_type=str, accelerator_count=int +): + """Returns machine spec to use for a given location. + + Args: + location: Where the machine will run. + use_test_spec: Whether to use a lower resource machine for testing. + + Returns: + Machine spec. + + Raises: + ValueError: If accelerators are requested in an unsupported location. + """ + outputs = NamedTuple( + 'MachineSpec', + machine_type=str, + accelerator_type=str, + accelerator_count=int, + ) + tpu_regions = {'europe-west4'} + gpu_regions = {'us-central1'} + if use_test_spec: + return outputs( + machine_type='a2-highgpu-1g', + accelerator_type='NVIDIA_TESLA_A100', + accelerator_count=1, + ) + elif location in tpu_regions: + return outputs( + machine_type='cloud-tpu', + accelerator_type='TPU_V3', + accelerator_count=64, + ) + elif location in gpu_regions: + return outputs( + machine_type='a2-ultragpu-8g', + accelerator_type='NVIDIA_A100_80GB', + accelerator_count=8, + ) + raise ValueError( + f'Unsupported accelerator location {location}. Must be one of' + f' {tpu_regions | gpu_regions}.' + ) + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_image_uri( + image_name: str, + project: str, + location: str, + artifact_registry: str, + image_name_prefix: str, + tag: str, + accelerator_type: str = '', + accelerator_count: int = 0, +) -> str: + """Generates image uri based on base image name and accelerator type. + + Args: + image_name: Base image name, e.g. ``'sft'`` or ``'reward_model'``. + project: Project that contains the artifact registry. + location: Region that contains the artifact registry. + artifact_registry: Registry that contains Docker images. + image_name_prefix: Text to prepend to the base image name. + tag: Image tag. + accelerator_type: One of the supported accelerator types, e.g. ``'TPU_V3'``. + accelerator_count: Number of accelerators. + + Returns: + Docker image uri + + Raises: + ValueError: if an unsupported accelerator type is provided. + """ + cpu_only_images = { + 'text_importer', + 'text_comparison_importer', + } + + if image_name in cpu_only_images: + accelerator_postfix = '' + elif accelerator_type == 'TPU_V3': + accelerator_postfix = '_tpu' + elif accelerator_type == 'NVIDIA_A100_80GB' and accelerator_count == 8: + accelerator_postfix = '_gpu_test' + else: + accelerator_postfix = '_gpu' + + backup_images = { + 'sft', + 'reward_model', + 'reinforcer', + 'infer', + 'text_importer', + 'text_comparison_importer', + } + if image_name in backup_images and accelerator_postfix != '_gpu_test': + accelerator_postfix += '_backup' + return f'{location}-docker.pkg.dev/{project}/{artifact_registry}/{image_name_prefix}{image_name}{accelerator_postfix}:{tag}' + + +# Resolves image uri from the environment's private artifact registry. +# By default this resolves an image in the vertex private registry. +resolve_private_image_uri = functools.partial( + resolve_image_uri, + project=env.PRIVATE_ARTIFACT_REGISTRY_PROJECT, + location=env.PRIVATE_ARTIFACT_REGISTRY_LOCATION, + artifact_registry=env.PRIVATE_ARTIFACT_REGISTRY, + image_name_prefix=env.PRIVATE_IMAGE_NAME_PREFIX, + tag=env.get_private_image_tag(), +) + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_data_paths( + input_dataset: str, +) -> NamedTuple('DataPaths', tfds_data_dir=str, tfds_name=str): + """Resolves dataset paths needed by downstream components.""" + # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported + import os + # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported + outputs = NamedTuple('DataPaths', tfds_data_dir=str, tfds_name=str) + tfds_data_dir, tfds_name = os.path.split(input_dataset) + return outputs( + tfds_data_dir=tfds_data_dir, + tfds_name=tfds_name, + ) + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_reference_model_metadata( + large_model_reference: str, + reference_model_path: Optional[str] = None, +) -> NamedTuple( + 'Outputs', + large_model_reference=str, + reference_model_path=str, + reward_model_reference=str, + reward_model_path=str, +): + """Resolves reference model metadata needed by downstream components. + + Args: + large_model_reference: User-provided reference model name. + reference_model_path: Optional path to a tuned based model to use in place + of the default base model. If specified, the model at this path must be a + tuned version of the base model associated with ``large_model_reference``. + + Returns: + Base model name (used by downstream components to find gin configs and load + vocabularies) and the path to the base model checkpoint. + + Raises: + ValueError: if no metadata exists for the given base model. + """ + reference_model_metadata = NamedTuple( + 'ReferenceModelMetadata', + large_model_reference=str, + reference_model_path=str, + reward_model_reference=str, + reward_model_path=str, + is_supported=bool, + ) + + reference_models = { + 't5-small': reference_model_metadata( + large_model_reference='T5_SMALL', + reference_model_path=( + 'gs://t5-data/pretrained_models/t5x/flan_t5_small/' + ), + reward_model_reference='T5_SMALL', + reward_model_path='gs://t5-data/pretrained_models/t5x/t5_1_1_small', + is_supported=True, + ), + 't5-large': reference_model_metadata( + large_model_reference='T5_LARGE', + reference_model_path=( + 'gs://t5-data/pretrained_models/t5x/flan_t5_large/' + ), + reward_model_reference='T5_LARGE', + reward_model_path='gs://t5-data/pretrained_models/t5x/t5_1_1_large', + is_supported=True, + ), + 't5-xl': reference_model_metadata( + large_model_reference='T5_XL', + reference_model_path='gs://t5-data/pretrained_models/t5x/flan_t5_xl/', + reward_model_reference='T5_XL', + reward_model_path='gs://t5-data/pretrained_models/t5x/t5_1_1_xl', + is_supported=True, + ), + 't5-xxl': reference_model_metadata( + large_model_reference='T5_XXL', + reference_model_path=( + 'gs://t5-data/pretrained_models/t5x/flan_t5_xxl/' + ), + reward_model_reference='T5_XL', + reward_model_path='gs://t5-data/pretrained_models/t5x/t5_1_1_xl', + is_supported=True, + ), + 'palm-tiny': reference_model_metadata( + large_model_reference='PALM_TINY', + reference_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_palm_tiny/', + reward_model_reference='PALM_TINY', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_palm_tiny/', + is_supported=False, + ), + 'gecko': reference_model_metadata( + large_model_reference='GECKO', + reference_model_path=( + 'gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_gecko/' + ), + reward_model_reference='GECKO', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_gecko_pretrain/', + is_supported=False, + ), + 'otter': reference_model_metadata( + large_model_reference='OTTER', + reference_model_path=( + 'gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_otter/' + ), + reward_model_reference='OTTER', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_otter_pretrain/', + is_supported=False, + ), + 'bison': reference_model_metadata( + large_model_reference='BISON', + reference_model_path=( + 'gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_bison/' + ), + reward_model_reference='OTTER', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_otter_pretrain/', + is_supported=False, # Deprecated: Use text-bision@001 instead. + ), + 'text-bison@001': reference_model_metadata( + large_model_reference='BISON', + reference_model_path=( + 'gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_bison/' + ), + reward_model_reference='OTTER', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_otter_pretrain/', + is_supported=True, + ), + 'chat-bison@001': reference_model_metadata( + large_model_reference='BISON', + reference_model_path=( + 'gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_bison/' + ), + reward_model_reference='OTTER', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_otter_pretrain/', + is_supported=True, + ), + 'elephant': reference_model_metadata( + large_model_reference='ELEPHANT', + reference_model_path=( + 'gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_elephant/' + ), + reward_model_reference='OTTER', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/palm/t5x_otter_pretrain/', + is_supported=False, + ), + 'llama-2-7b': reference_model_metadata( + large_model_reference='LLAMA_2_7B', + reference_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_7b/', + reward_model_reference='LLAMA_2_7B', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_7b/', + is_supported=True, + ), + 'llama-2-13b': reference_model_metadata( + large_model_reference='LLAMA_2_13B', + reference_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_13b/', + reward_model_reference='LLAMA_2_13B', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_13b/', + is_supported=True, + ), + 'llama-2-7b-chat': reference_model_metadata( + large_model_reference='LLAMA_2_7B_CHAT', + reference_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_7b_chat/', + reward_model_reference='LLAMA_2_7B', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_7b/', + is_supported=True, + ), + 'llama-2-13b-chat': reference_model_metadata( + large_model_reference='LLAMA_2_13B_CHAT', + reference_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_13b_chat/', + reward_model_reference='LLAMA_2_13B', + reward_model_path='gs://vertex-rlhf-restricted/pretrained_models/llama/t5x_llama_2_13b/', + is_supported=True, + ), + } + + reference_model_key = large_model_reference.lower().replace('_', '-') + if reference_model_key not in reference_models: + supported_models = [ + k for k, v in reference_models.items() if v.is_supported + ] + raise ValueError( + f'Unknown reference model {large_model_reference}.' + ' large_model_reference must be one of' + f' {sorted(supported_models)}.' + ) + + reference_model = reference_models[reference_model_key] + + outputs = NamedTuple( + 'Outputs', + large_model_reference=str, + reference_model_path=str, + reward_model_reference=str, + reward_model_path=str, + ) + + return outputs( + large_model_reference=reference_model.large_model_reference, + reference_model_path=( + reference_model_path or reference_model.reference_model_path + ), + reward_model_reference=reference_model.reward_model_reference, + reward_model_path=reference_model.reward_model_path, + ) + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def convert_to_delimited_string(items: List[str], delimiter: str = ',') -> str: + """Converts a list of strings to single string delimited by the specified character.""" + return delimiter.join(items) + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def generate_default_instruction( + task: str, + target_sequence_length: int, + instruction_override: str = '', +) -> str: + """Generates a default instruction if no override is provided.""" + if instruction_override: + return instruction_override + task = task.lower() + if task == 'summarization': + return f'Summarize in less than {target_sequence_length} words.' + + elif task == 'question_answer': + return f'Answer the question in less than {target_sequence_length} words.' + + else: + raise ValueError( + f'Task not recognized: {task}. Supported tasks are: "summarization",' + ' "question_answer".' + ) + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_upload_location(upload_location: Optional[str] = None) -> str: + """Gets the region to upload the model. + + Args: + upload_location: User-specified region to upload the model to. + + Returns: + Where to upload the model. If no location is specified, the model will be + uploaded to the region where the pipeline is running. + """ + # pylint: disable=g-import-not-at-top + import os + # pylint: enable=g-import-not-at-top + return upload_location or os.environ['CLOUD_ML_REGION'] + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_regional_endpoint(upload_location: str) -> str: + """Gets the regional endpoint used to upload a model to the registry. + + Args: + upload_location: Region where the model will be uploaded. + + Returns: + Regional endpoint. + """ + return f'https://{upload_location}-aiplatform.googleapis.com/ui' + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_model_display_name( + large_model_reference: str, + model_display_name: Optional[str] = None, +) -> str: + """Gets the model display name shown in the registry and used for endpoints. + + Args: + large_model_reference: Base model tuned by the pipeline. + model_display_name: User-provided display name. If not provided, a default + display name will be created. + + Returns: + Either the user-provided name or a default display name with the form + ``{large_model_reference}-{timestamp}`` + """ + # pylint: disable=g-import-not-at-top + import datetime + # pylint: enable=g-import-not-at-top + now = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + return model_display_name or f'{large_model_reference.lower()}-{now}' + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_deploy_model( + deploy_model: bool, large_model_reference: str +) -> bool: + """Resolves runtime parameter that determines whether the tuned model should be deployed.""" + supported_models = {'BISON'} + if deploy_model and large_model_reference in supported_models: + return True + return False + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def value_exists(value: Optional[str] = None) -> bool: + """Returns whether a runtime parameter was provided. + + Args: + value: That might have been provided. + + Returns: + Whether the string is not None and non-empty. + """ + if not value: + return False + return True + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_candidate_columns( + candidate_columns: Optional[List[str]] = None, +) -> List[str]: + """Returns candidate columns provided by the user or the default: ['candidate_0', 'candidate_1'].""" + return candidate_columns or ['candidate_0', 'candidate_1'] + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_upload_model(large_model_reference: str) -> bool: + """Returns whether the model should be uploaded.""" + supported_models = {'BISON'} + if large_model_reference in supported_models: + return True + return False + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def resolve_instruction( + large_model_reference: str, instruction: Optional[str] = None +) -> str: + """Resolves the instruction to use for a given reference model. + + Args: + large_model_reference: Base model tuned by the pipeline. + instruction: Instruction provided at runtime. + + Returns: + Instruction to use during tokenization based on model type. Returns an empty + string for chat models because the instruction is prepended as the default + context. Otherwise the original instruction is returned. + """ + instruction = instruction or '' + return instruction if 'chat' not in large_model_reference.lower() else '' diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/preprocess_chat_dataset.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/preprocess_chat_dataset.py new file mode 100644 index 0000000000..e4e5697e37 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/preprocess_chat_dataset.py @@ -0,0 +1,250 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Component the preprocesses chat dataset before tokenization.""" + +from google_cloud_pipeline_components import _image +from kfp import dsl + + +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def preprocess_chat_dataset( + large_model_reference: str, + input_dataset_uri: str, + processed_dataset: dsl.OutputPath(dsl.Artifact), # pytype: disable=invalid-annotation + processed_dataset_uri: dsl.OutputPath(str), # pytype: disable=invalid-annotation + default_context: str = '', + allow_local_files: bool = False, +): # pylint: disable=g-doc-args + # fmt: off + """Preprocesses datasets before tokenization. + + For text datasets, this is a no-op. + + Args: + large_model_reference: Name of the base model. Supported values are `text-bison@001`, `chat-bison@001`, `t5-small`, `t5-large`, `t5-xl` and `t5-xxl`. `text-bison@001`, `chat-bison@001` and `t5-small` are supported in ``us-central1` and `europe-west4`. `t5-large`, `t5-xl` and `t5-xxl` are only supported in `europe-west4`. + input_dataset_uri: Path to an unprocessed JSONL dataset. + default_context: Default context to apply to each example if a chat model is specified. + allow_local_files: Whether input URIs can specify local file paths. + + Returns: + processed_dataset: Processed chat dataset. Each example will contain fields `input_text` and `output_text`. + processed_dataset_uri: String pattern that can be used to find the processed dataset in downstream components. + + """ + # fmt: on + # pylint: disable=g-import-not-at-top + import dataclasses + import json + import os + from typing import Any, Callable, List, Mapping + import apache_beam as beam + # pylint: enable=g-import-not-at-top + + # [ Define helper methods and classes for preprocessing + # pylint: disable=invalid-name + INPUT_TEXT_KEY = 'input_text' + OUTPUT_TEXT_KEY = 'output_text' + CONTEXT_KEY = 'context' + MESSAGES_KEY = 'messages' + AUTHOR_KEY = 'author' + CONTENT_KEY = 'content' + AUTHOR_USER = 'user' + AUTHOR_ASSISTANT = 'assistant' + VALID_AUTHORS = {AUTHOR_USER, AUTHOR_ASSISTANT} + + # pylint: enable=invalid-name + @dataclasses.dataclass + class PromptSchema: + global_prefix: str + user_prefix: str + user_postfix: str + assistant_prefix: str + assistant_postfix: str + get_system_message: Callable[[str], str] # pytype: disable=invalid-annotation + + def _get_chat_bison_001_system_message(context: str) -> str: + return f'[SYSTEM]:{context}\n\n' if context else '' + + chat_bison_001_schema = PromptSchema( + global_prefix=( + 'Only answer after [assistant] and never reply as [user]:\n' + ), + get_system_message=_get_chat_bison_001_system_message, + user_prefix='[user]:', + user_postfix='\n', + assistant_prefix='[assistant]:', + assistant_postfix='\n', + ) + + def _get_chat_llama_system_message(context: str) -> str: + return f'<>\n{context}\n<>\n\n' if context else '' + + chat_llama_schema = PromptSchema( + global_prefix='[INST] ', + get_system_message=_get_chat_llama_system_message, + user_prefix='', + user_postfix=' [/INST]', + assistant_prefix=' ', + assistant_postfix='[INST] ', + ) + + MODEL_TO_SCHEMA_MAPPING = { # pylint: disable=invalid-name + 'chat-bison@001': chat_bison_001_schema, + 'llama-2-7b-chat': chat_llama_schema, + 'llama-2-13b-chat': chat_llama_schema, + } + + def get_gcs_path(input_path: str, allow_local_files: bool) -> str: + """Gets the /gcs/ path for a given URI.""" + if input_path.startswith('gs://'): + return input_path.replace('gs://', '/gcs/', 1) + elif input_path.startswith('/gcs/') or allow_local_files: + return input_path + else: + raise ValueError( + f'Invalid Cloud storage URI {input_path}. ' + 'Must start with `gs://` or `/gcs/`.' + ) + + def get_gs_path(input_path: str, allow_local_files: bool) -> str: + """Gets the gs:// path for a given URI.""" + if input_path.startswith('/gcs/'): + return input_path.replace('/gcs/', 'gs://', 1) + elif input_path.startswith('gs://') or allow_local_files: + return input_path + else: + raise ValueError( + f'Invalid Cloud storage URI {input_path}. ' + 'Must start with `gs://` or `/gcs/`.' + ) + + class JsonCoder(beam.coders.Coder): + """A coder that encodes/decodes lines as JSON strings.""" + + def encode(self, x): + return json.dumps(x).encode('utf-8') + + def decode(self, x): + return json.loads(x) + + class ChatDatasetProcessor(beam.DoFn): + """Converts chat data from input format to the format expected by the model.""" + + def __init__(self, default_context: str, prompt_schema: PromptSchema): + self._default_context = default_context + self._schema = prompt_schema + + def _get_messages_or_fail( + self, element: Mapping[str, Any] + ) -> List[Mapping[str, str]]: + messages = element.get(MESSAGES_KEY) + if not messages or len(messages) <= 1: + raise ValueError( + 'Chat messages length should be greater than 1. Please include a ' + f'`messages` field in each line of dataset: {element}.' + ) + return messages + + def _get_author_or_fail(self, message: Mapping[str, str]) -> str: + author = message.get(AUTHOR_KEY) + if not author or author not in VALID_AUTHORS: + raise ValueError( + 'The `author` of each message needs to be from one of' + f' {VALID_AUTHORS}. Got author = {author}.' + ) + return author + + def _get_content_or_fail(self, message: Mapping[str, str]) -> str: + content = message.get(CONTENT_KEY) + if not content: + raise ValueError( + 'The `content` of each message needs to be non-empty. ' + f'Invalid message: {message}' + ) + return content + + def process(self, element): + context = element.get(CONTEXT_KEY, self._default_context) + messages = self._get_messages_or_fail(element) + + message_history = [ + self._schema.global_prefix, + self._schema.get_system_message(context), + ] + for message in messages: + author = self._get_author_or_fail(message) + content = self._get_content_or_fail(message) + if author == AUTHOR_USER: + message_history.append( + f'{self._schema.user_prefix}{content}{self._schema.user_postfix}' + ) + elif author == AUTHOR_ASSISTANT: + message_history.append(self._schema.assistant_prefix) + input_text = ''.join(message_history) + yield {INPUT_TEXT_KEY: input_text.rstrip(), OUTPUT_TEXT_KEY: content} + message_history = [ + input_text, + f'{content}{self._schema.assistant_postfix}', + ] + else: + raise ValueError( + f'Unknown author {author}. Must be one of {VALID_AUTHORS}.' + ) + + # ] + + processed_dataset_uri = get_gcs_path(processed_dataset_uri, allow_local_files) + + # Reuse the input dataset if no preprocessing is needed. + if large_model_reference.lower() not in MODEL_TO_SCHEMA_MAPPING: + with open(processed_dataset_uri, 'w') as f: + f.write(input_dataset_uri) + return + + prompt_schema = MODEL_TO_SCHEMA_MAPPING[large_model_reference] + + # Provide gs:// paths for datasets processed by Beam. + input_dataset_uri = get_gs_path(input_dataset_uri, allow_local_files) + processed_dataset = get_gs_path(processed_dataset, allow_local_files) + os.makedirs(processed_dataset, exist_ok=True) + processed_dataset_prefix = os.path.join(processed_dataset, 'shard') + + pipeline_options = ( + beam.options.pipeline_options.PipelineOptions.from_dictionary({ + 'runner': 'DirectRunner', + }) + ) + with beam.Pipeline(options=pipeline_options) as pipeline: + _ = ( + pipeline + | 'Read JSON from input dataset' + >> beam.io.ReadFromText(input_dataset_uri, coder=JsonCoder()) + | 'Process chat dataset' + >> beam.ParDo( + ChatDatasetProcessor( + default_context=default_context, prompt_schema=prompt_schema + ) + ) + | 'Write processed JSON to output file' + >> beam.io.WriteToText( + file_path_prefix=processed_dataset_prefix, + file_name_suffix='.jsonl', + coder=JsonCoder(), + ) + ) + + # Write file pattern that the tokenizer can use to find all processed files. + with open(processed_dataset_uri, 'w') as f: + processed_dataset_pattern = os.path.join(processed_dataset, '*.jsonl') + f.write(processed_dataset_pattern) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/private_text_comparison_importer.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/private_text_comparison_importer.py new file mode 100644 index 0000000000..3c81443af9 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/private_text_comparison_importer.py @@ -0,0 +1,86 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Container component that imports Tensorflow Datasets.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.llm import utils +import kfp + + +@kfp.dsl.container_component +def PrivateTextComparisonImporter( # pylint: disable=invalid-name + project: str, + location: str, + input_text: str, + inputs_field_name: str, + comma_separated_candidates_field_names: str, + choice_field_name: str, + split: str, + large_model_reference: str, + image_uri: str, + output_dataset_path: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + gcp_resources: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + machine_type: str = 'e2-highmem-8', + instruction: str = '', +) -> kfp.dsl.ContainerSpec: # pylint: disable=g-doc-args + """Import a text dataset. + + Args: + project: Project used to run the job. + location: Location used to run the job. + input_text: Path to text data. Supports glob patterns. + inputs_field_name: Name of field that contains input text. + comma_separated_candidates_field_names: Comma separated list of fields that + contain candidate text, e.g. ``'field_1,field_2,field_3'``. + choice_field_name: Name of field that specifies the index of the best + candidate. + split: The created seqio task has 1 split, its name is specified by this + argument. + large_model_reference: Predefined model used to create the model to be + trained. This paramerter is used for obtaining model vocabulary because + this component tokenizes and then caches the tokenized tasks. + machine_type: The type of the machine to provision for the custom job. + instruction: Optional instruction to prepend to inputs field. + image_uri: Location of the text comparison importer image. + dataflow_worker_image_uri: Location of the Dataflow worker image. + + Returns: + output_dataset_path: Path to cached SeqIO task created from input dataset. + gcp_resources: GCP resources that can be used to track the custom job. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_payload( + display_name='TfdsComparisonImporter', + machine_type=machine_type, + image_uri=image_uri, + args=[ + f'--input_text={input_text}', + f'--inputs_field_name={inputs_field_name}', + f'--comma_separated_candidates_field_names={comma_separated_candidates_field_names}', + f'--choice_field_name={choice_field_name}', + f'--split={split}', + f'--output_cache_dir={output_dataset_path}', + f'--instruction={instruction}', + f'--large_model_reference={large_model_reference}', + ( + '--private_bucket_subdir=' + f'{kfp.dsl.PIPELINE_TASK_NAME_PLACEHOLDER}_' + f'{kfp.dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + ], + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/private_text_importer.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/private_text_importer.py new file mode 100644 index 0000000000..36d7d4986a --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/private_text_importer.py @@ -0,0 +1,94 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Container component that imports Tensorflow Datasets.""" +import os +from typing import Optional + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.llm import utils +from kfp import dsl + + +def _resolve_image(default: str = '') -> str: + return os.environ.get('TEXT_IMPORTER_IMAGE_OVERRIDE') or default + + +# pytype: disable=unsupported-operands +@dsl.container_component +def PrivateTextImporter( # pylint: disable=invalid-name + project: str, + location: str, + input_text: str, + inputs_field_name: str, + targets_field_name: str, + large_model_reference: str, + imported_data: dsl.Output[dsl.Dataset], # pylint: disable=unused-argument + imported_data_path: dsl.OutputPath(str), # pytype: disable=invalid-annotation + gcp_resources: dsl.OutputPath(str), # pytype: disable=invalid-annotation + instruction: str = '', + image_uri: str = utils.get_default_image_uri('text_importer_backup'), + machine_type: str = 'e2-highmem-8', + output_split_name: str = 'all', + max_num_input_examples: Optional[int] = None, +) -> dsl.ContainerSpec: # pylint: disable=g-doc-args + """Import a text dataset. + + Args: + project: Project used to run the job. + location: Location used to run the job. + input_text: Path to text data. Supports glob patterns. + inputs_field_name: Name of field that contains input text. + targets_field_name: Name of field that contains target text. + large_model_reference: Predefined model used to create the model to be + trained. This paramerter is used for obtaining model vocabulary because + this component tokenizes and then caches the tokenized tasks. + instruction: Optional instruction to prepend to inputs field. + image_uri: Optional location of the text importer image. + machine_type: The type of the machine to provision for the custom job. + output_split_name: The created seqio task has 1 split, its name is specified + by this argument. + max_num_input_examples: Maximum number of examples to import. + + Returns: + imported_data: Artifact representing the imported data and cached Tasks. + imported_data_path: Path to cached SeqIO task created from input dataset. + gcp_resources: Tracker for GCP resources created by this component. + """ + subdir = ( + f'{dsl.PIPELINE_TASK_NAME_PLACEHOLDER}_{dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ) + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_payload( + display_name='TextImporter', + machine_type=machine_type, + image_uri=_resolve_image(image_uri), + args=[ + f'--input_text={input_text}', + f'--inputs_field_name={inputs_field_name}', + f'--targets_field_name={targets_field_name}', + f'--output_split_name={output_split_name}', + f'--instruction={instruction}', + f'--large_model_reference={large_model_reference}', + f'--private_bucket_subdir={subdir}', + f'--output_dataset_path={dsl.PIPELINE_ROOT_PLACEHOLDER}{subdir}', + f'--imported_data_path={imported_data_path}', + f'--max_num_input_examples={max_num_input_examples}', + '--executor_input={{$.json_escape[1]}}', + ], + ), + gcp_resources=gcp_resources, + ) +# pytype: enable=unsupported-operands diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/reinforcer.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/reinforcer.py new file mode 100644 index 0000000000..82d0ab2543 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/reinforcer.py @@ -0,0 +1,116 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP container component that performs reinforcement learning.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.llm import utils +import kfp + + +@kfp.dsl.container_component +def Reinforcer( # pylint: disable=invalid-name + project: str, + location: str, + train_steps: int, + accelerator_type: str, + accelerator_count: int, + large_model_reference: str, + reward_model_reference: str, + machine_type: str, + image_uri: str, + inputs_sequence_length: int, + targets_sequence_length: int, + input_reference_model_path: str, + input_reward_model_path: str, + input_dataset_path: str, + output_model_path: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + output_adapter_path: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + tensorboard_metrics: kfp.dsl.Output[kfp.dsl.Artifact], # pytype: disable=unsupported-operands + gcp_resources: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + train_split: str = 'train', + batch_size: int = 64, + learning_rate_multiplier: float = 1.0, + kl_coeff: float = 0.1, + lora_dim: int = 0, +) -> kfp.dsl.ContainerSpec: # pylint: disable=g-doc-args + """Trains a model using reinforcement learning. + + Args: + project: Project used to run the job. + location: Location used to run the job. + input_reference_model_path: Path to the base model to fine tune. + input_reward_model_path: Path to the reward model to use during + reinforcement learning. + input_dataset_path: Path to training dataset. + train_steps: Number of training steps. These are the number of steps + on top of any steps used to train the base model. + targets_length: Maximum decoder steps. Outputs will be at most this length. + accelerator_type: Type of TPU accelerator. Can be either TPU_V2 or TPU_V3. + accelerator_count: Number of TPU accelerators. + large_model_reference: Predefined model used to create the + ``input_reference_model``. + machine_type: The type of the machine to provision for the custom job. Must + be a valid GCE instance type and compatible with the accelerator type. + image_uri: Location of reinforcement learning Docker image. + inputs_sequence_length: Maximum number of input tokens per row. + targets_sequence_length: Maximum number of target tokens per row. + train_split: Name of the split in the input dataset that contains training + data. Default is ``'train'``. + batch_size: Number of examples in each finetuning step. Default is 64. + kl_coeff: Coefficient for KL penalty. This regularizes the policy model and + penalizes if it diverges from its initial distribution. If set to 0, then + the reference LM is not loaded into memory. + lora_dim: The rank of the LoRA adapter. If >0, then use LoRA-tuning. If =0, + then use full-tuning. + learning_rate_multiplier: Constant multiplied by the base learning rate used + to adjust the learning rate during reinforcement learning. + + Returns: + output_model_path: Path to the trained model checkpoint. + output_adapter_path: Path to the trained model adapter if LoRA tuning was + used. + tensorboard_metrics: Training stats (tensorboard) path. + gcp_resources: GCP resources that can be used to track the custom finetuning + job. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_payload( + display_name='Reinforcer', + accelerator_type=accelerator_type, + accelerator_count=accelerator_count, + machine_type=machine_type, + image_uri=image_uri, + args=[ + f'--input_reference_model_path={input_reference_model_path}', + f'--input_reward_model_path={input_reward_model_path}', + f'--input_dataset_path={input_dataset_path}', + f'--train_steps={train_steps}', + f'--output_model_path={output_model_path}', + f'--output_adapter_path={output_adapter_path}', + f'--tensorboard_metrics_path={tensorboard_metrics.path}', + f'--large_model_reference={large_model_reference}', + f'--reward_model_reference={reward_model_reference}', + f'--inputs_sequence_length={inputs_sequence_length}', + f'--targets_sequence_length={targets_sequence_length}', + f'--train_split={train_split}', + f'--batch_size={batch_size}', + f'--learning_rate_multiplier={learning_rate_multiplier}', + f'--kl_coeff={kl_coeff}', + f'--lora_dim={lora_dim}', + ], + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/reward_model_trainer.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/reward_model_trainer.py new file mode 100644 index 0000000000..cdf0bba4db --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/reward_model_trainer.py @@ -0,0 +1,104 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP container component that trains a reward model.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.llm import utils +import kfp + + +@kfp.dsl.container_component +def RewardModelTrainer( # pylint: disable=invalid-name + project: str, + location: str, + train_steps: int, + accelerator_type: str, + accelerator_count: int, + large_model_reference: str, + machine_type: str, + image_uri: str, + inputs_sequence_length: int, + targets_sequence_length: int, + input_model_path: str, + input_dataset_path: str, + output_model_path: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + tensorboard_metrics: kfp.dsl.Output[kfp.dsl.Artifact], # pytype: disable=unsupported-operands + gcp_resources: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + train_split: str = 'train', + batch_size: int = 64, + learning_rate_multiplier: float = 1.0, + lora_dim: int = 0, +) -> kfp.dsl.ContainerSpec: # pylint: disable=g-doc-args + """Trains a reward model. + + Args: + project: Project used to run the job. + location: Location used to run the job. + input_model_path: Path to the base model to fine tune. + input_dataset_path: Path to dataset to use to train a reward model. + train_steps: Number of training steps. These are the number of steps + on top of any steps used to train the base model. + accelerator_type: Type of TPU accelerator. Can be either TPU_V2 or TPU_V3. + accelerator_count: Number of TPU accelerators. + large_model_reference: Predefined model used to create the ``input_model``. + machine_type: The type of the machine to provision for the custom job. Must + be a valid GCE instance type and compatible with the accelerator type. + image_uri: Location of reward model Docker image. + inputs_sequence_length: Maximum number of input tokens per row. + targets_sequence_length: Maximum number of target tokens per row. + train_split: Name of the split in the input dataset that contains training + data. Default is ``'train'``. + batch_size: Number of examples in each finetuning step. Default is 64. + lora_dim: The rank of the LoRA adapter. If >0, then use LoRA-tuning. If =0, + then use full-tuning. + learning_rate_multiplier: Constant multiplied by the base learning rate used + to adjust the learning rate when training a reward model. + + Returns: + output_model: Trained reward model. + tensorboard_metrics: Training stats (tensorboard) path. + gcp_resources: GCP resources that can be used to track the custom finetuning + job. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_payload( + display_name='RewardModelTrainer', + accelerator_type=accelerator_type, + accelerator_count=accelerator_count, + machine_type=machine_type, + image_uri=image_uri, + args=[ + f'--train_steps={train_steps}', + f'--input_model_path={input_model_path}', + f'--input_dataset_path={input_dataset_path}', + f'--output_model_path={output_model_path}', + f'--tensorboard_metrics_path={tensorboard_metrics.path}', + f'--large_model_reference={large_model_reference}', + f'--inputs_sequence_length={inputs_sequence_length}', + f'--targets_sequence_length={targets_sequence_length}', + f'--train_split={train_split}', + f'--batch_size={batch_size}', + f'--learning_rate_multiplier={learning_rate_multiplier}', + ( + '--private_bucket_subdir=' + f'{kfp.dsl.PIPELINE_TASK_NAME_PLACEHOLDER}_' + f'{kfp.dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + f'--lora_dim={lora_dim}', + ], + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/supervised_fine_tuner.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/supervised_fine_tuner.py new file mode 100644 index 0000000000..7254ce2407 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/supervised_fine_tuner.py @@ -0,0 +1,105 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Container component that performs supervised fine tuning.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.llm import utils +import kfp + + +@kfp.dsl.container_component +def SupervisedFineTuner( # pylint: disable=invalid-name + project: str, + location: str, + train_steps: int, + inputs_sequence_length: int, + targets_sequence_length: int, + accelerator_type: str, + accelerator_count: int, + large_model_reference: str, + machine_type: str, + image_uri: str, + input_model_path: str, + input_dataset_path: str, + output_model_path: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + tensorboard_metrics: kfp.dsl.Output[kfp.dsl.Artifact], # pytype: disable=unsupported-operands + gcp_resources: kfp.dsl.OutputPath(str), # pytype: disable=invalid-annotation + train_split: str = 'train', + batch_size: int = 64, + learning_rate_multiplier: float = 1.0, + lora_dim: int = 0, +) -> kfp.dsl.ContainerSpec: # pylint: disable=g-doc-args + """Performs supervised fine tuning. + + Args: + project: Project used to run the job. + location: Location used to run the job. + input_model_path: Path to the base model to fine tune. + train_steps: Number of training steps. These are the number of steps on top + of any steps used to train the base model. + accelerator_type: Type of TPU accelerator. Can be either TPU_V2 or TPU_V3. + accelerator_count: Number of TPU accelerators. + input_dataset_path: Path to training dataset + large_model_reference: Predefined model used to create the ``input_model``. + machine_type: The type of the machine to provision for the custom job. Must + be a valid GCE instance type and compatible with the accelerator type. + image_uri: Location of reinforcement learning Docker image. + inputs_sequence_length: Maximum number of input tokens per row. + targets_sequence_length: Maximum number of target tokens per row. + train_split: Name of the split in the input dataset that contains training + data. Default is ``'train'``. + batch_size: The batch size is the number of training examples used to train + a single forward and backward pass. Default is 64. + lora_dim: The rank of the LoRA adapter. If >0, then use LoRA-tuning. If =0, + then use full-tuning. + learning_rate_multiplier: Constant multiplied by the base learning rate used + to adjust the learning rate during supervised fine tuning. + + Returns: + output_model_path: Fine-tuned model path. + tensorboard_metrics: Training stats (tensorboard) path. + gcp_resources: GCP resources that can be used to track the custom finetuning + job. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_payload( + display_name='SupervisedFineTuner', + accelerator_type=accelerator_type, + accelerator_count=accelerator_count, + machine_type=machine_type, + image_uri=image_uri, + args=[ + f'--input_model_path={input_model_path}', + f'--train_steps={train_steps}', + f'--inputs_sequence_length={inputs_sequence_length}', + f'--targets_sequence_length={targets_sequence_length}', + f'--input_dataset_path={input_dataset_path}', + f'--output_model_path={output_model_path}', + f'--tensorboard_metrics_path={tensorboard_metrics.path}', + f'--large_model_reference={large_model_reference}', + f'--train_split={train_split}', + f'--batch_size={batch_size}', + f'--learning_rate_multiplier={learning_rate_multiplier}', + ( + '--private_bucket_subdir=' + f'{kfp.dsl.PIPELINE_TASK_NAME_PLACEHOLDER}_' + f'{kfp.dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + f'--lora_dim={lora_dim}', + ], + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_llm_model.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_llm_model.py new file mode 100644 index 0000000000..da566b7d45 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_llm_model.py @@ -0,0 +1,128 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Component for upload_llm_model.""" + +from google_cloud_pipeline_components import _image +from kfp import dsl + + +# pylint: disable=g-import-not-at-top, invalid-name, +# pylint: disable=g-doc-args +# pytype: disable=invalid-annotation +# pytype: disable=unsupported-operands +# pytype: disable=import-error +@dsl.component(base_image=_image.GCPC_IMAGE_TAG, install_kfp_package=False) +def upload_llm_model( + project: str, + location: str, + artifact_uri: dsl.Input[dsl.Artifact], + model_reference_name: str, + model_display_name: str, + regional_endpoint: str, + model_resource_name: dsl.OutputPath(str), + gcp_resources: dsl.OutputPath(str), + encryption_spec_key_name: str = '', + upload_model: bool = True, +): + """Uploads LLM model. + + Args: + project: Name of the GCP project. + location: Location for model upload and deployment. + artifact_uri: KFP Artifact for adapter. + model_reference_name: Large model reference name. + model_display_name: Name of the model (shown in Model Registry). + regional_endpoint: Regional API endpoint. + encryption_spec_key_name: Customer-managed encryption key. + upload_model: Whether to upload the model to the Model Registry. Default + is ``True``. If ``False``, the model will not be uploaded and output + artifacts will contain empty strings. + + Returns: + model_resource_name: Path to the created Model on Model Registry. + gcp_resources: Serialized JSON of `gcp_resources`. + """ + import json + import logging + import os + import sys + + try: + from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner + except ImportError: + from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner + + try: + os.makedirs(os.path.dirname(model_resource_name), exist_ok=True) + + if not upload_model: + with open(model_resource_name, 'w') as fout: + fout.write('') + return + + pipeline_labels_str = os.getenv('VERTEX_AI_PIPELINES_RUN_LABELS') + labels = json.loads(pipeline_labels_str) if pipeline_labels_str else {} + labels['google-vertex-llm-tuning-base-model-id'] = ( + model_reference_name.replace('@', '-') + ) + + model_upload_payload = { + 'model': { + 'displayName': model_display_name, + 'largeModelReference': {'name': model_reference_name}, + 'labels': labels, + 'generatedModelSource': {'genie_source': {'base_model_uri': ''}}, + 'artifactUri': artifact_uri.uri, + } + } + if encryption_spec_key_name: + model_upload_payload['model']['encryption_spec'] = { + 'kms_key_name': encryption_spec_key_name + } + + regional_endpoint = regional_endpoint.rstrip('/') + upload_model_uri = ( + f'{regional_endpoint}/projects/{project}/locations/{location}/models:' + 'upload' + ) + + remote_runner = lro_remote_runner.LroRemoteRunner(location) + upload_model_lro = remote_runner.create_lro( + upload_model_uri, + json.dumps(model_upload_payload), + gcp_resources, + ) + upload_model_lro = remote_runner.poll_lro(lro=upload_model_lro) + model_resource = upload_model_lro['response']['model'] + model_version_id = upload_model_lro['response'].get( + 'model_version_id' + ) or upload_model_lro['response'].get('modelVersionId') + if model_version_id: + model_resource += f'@{model_version_id}' + + with open(model_resource_name, 'w') as fout: + fout.write(model_resource) + + except Exception as e: # pylint: disable=broad-exception-caught + if isinstance(e, ValueError): + raise + logging.exception(str(e)) + sys.exit(13) + + +# pytype: enable=import-error +# pytype: enable=unsupported-operands +# pytype: enable=invalid-annotation +# pylint: enable=g-doc-args +# pylint: enable=g-import-not-at-top, invalid-name, broad-exception-caught diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_tensorboard_metrics.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_tensorboard_metrics.py new file mode 100644 index 0000000000..caa90e8bc3 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/upload_tensorboard_metrics.py @@ -0,0 +1,122 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""KFP Component that uploads tensorboard metrics.""" + +from google_cloud_pipeline_components import _image +from kfp import dsl + +_TB_UPLOADER_SHELL_SCRIPT = """ +set -e -x +TENSORBOARD_RESOURCE_ID="$0" +METRICS_DIRECTORY_URI="$1" +EXPERIMENT_NAME="$2" +TENSORBOARD_URI="$3" + +mkdir -p "$(dirname ${TENSORBOARD_URI})" +if [ -z "${TENSORBOARD_RESOURCE_ID}" ]; +then + echo "TensorBoard ID is not set. Skip uploading the TensorBoard." + echo -n "" > "${TENSORBOARD_URI}" + exit 0 +fi + +if [ -z "${METRICS_DIRECTORY_URI}" ]; then + echo "Metrics directory uri is not set." + exit 1 +elif [ -z "${EXPERIMENT_NAME}" ]; then + echo "Experiment name is not set." + exit 1 +elif [ -z "${TENSORBOARD_URI}" ]; then + echo "TensorBoard URI is not set." + exit 1 +fi + +case "${METRICS_DIRECTORY_URI}" in + "gs://"*) ;; + "/gcs/"*) + METRICS_DIRECTORY_URI=${METRICS_DIRECTORY_URI/"/gcs/"/"gs://"} + echo "Replaced /gcs/ path with ${METRICS_DIRECTORY_URI}" + ;; + *) + echo "Invalid metrics directory uri. Metrics directory uri must start with gs:// or /gcs/." + exit 1 + ;; +esac + +if [[ "${TENSORBOARD_RESOURCE_ID}" =~ ^projects/[^/]+/locations/[^/]+/tensorboards/[0-9]+$ ]]; then + echo "Split tensorboard resource id" + TENSORBOARD_RESOURCE_ARR=(${TENSORBOARD_RESOURCE_ID//\\// }) + PROJECT=${TENSORBOARD_RESOURCE_ARR[1]} + LOCATION=${TENSORBOARD_RESOURCE_ARR[3]} + TENSORBOARD_ID=${TENSORBOARD_RESOURCE_ARR[5]} +else + echo '[ERROR]: Invalid format of tensorboard_resource_id. It must be a string with format projects/${PROJECT_NUMBER}/locations/${LOCATION}/tensorboards/${TENSORBOARD_ID}' + exit 1 +fi + +set +e + +/opt/conda/bin/tb-gcp-uploader --tensorboard_resource_name \\ + "${TENSORBOARD_RESOURCE_ID}" \\ + --logdir="${METRICS_DIRECTORY_URI}" \\ + --experiment_name="${EXPERIMENT_NAME}" \\ + --one_shot=True + +if [ $? -ne 0 ]; then + exit 13 +fi + +set -e + +web_server_uri="tensorboard.googleusercontent.com" +tensorboard_resource_name_uri="projects+${PROJECT}+locations+${LOCATION}+tensorboards+${TENSORBOARD_ID}+experiments+${EXPERIMENT_NAME}" +echo -n "https://${LOCATION}.${web_server_uri}/experiment/${tensorboard_resource_name_uri}" > "${TENSORBOARD_URI}" +""" + + +@dsl.container_component +def upload_tensorboard_metrics( + tensorboard_resource_id: str, + experiment_name: str, + metrics_directory: dsl.Input[dsl.Artifact], # pytype: disable=unsupported-operands + tensorboard_uri: dsl.OutputPath(str), # pytype: disable=invalid-annotation +) -> dsl.ContainerSpec: + # fmt: off + # pylint: disable=g-doc-args + """Uploads tensorboard metrics. + + Args: + tensorboard_resource_id: TensorBoard resource ID in the form `projects/{project_number}/locations/{location}/tensorboards/{tensorboard_id}`. + experiment_name: Name of this tensorboard experiment. Must be unique to a given `projects/{project_number}/locations/{location}/tensorboards/{tensorboard_id}`. + metrics_directory_uri: Cloud storage location of the TensorBoard logs. + + Returns: + tensorboard_uri: URI of the uploaded tensorboard experiment. + """ + # pylint: enable=g-doc-args + # fmt: on + return dsl.ContainerSpec( + image='us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-11:latest', + command=[ + 'bash', + '-c', + _TB_UPLOADER_SHELL_SCRIPT, + ], + args=[ + tensorboard_resource_id, + metrics_directory.path, + experiment_name, + tensorboard_uri, + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py new file mode 100644 index 0000000000..2c5a6369bc --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils.py @@ -0,0 +1,121 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utility functions used to create custom Kubeflow components.""" +import os +from typing import Any, Dict, List + +from google_cloud_pipeline_components._implementation.llm import env +import kfp + + +def build_payload( + *, + display_name: str, + machine_type: str, + image_uri: str, + args: List[str], + accelerator_type: str = '', + accelerator_count: int = 0, + encryption_spec_key_name: str = '', +) -> Dict[str, Any]: + """Generates payload for a custom training job. + + Args: + display_name: Component display name. Can contain up to 128 UTF-8 + characters. + machine_type: The type of the machine to provision for the custom job. Must + be a valid GCE instance type and compatible with the accelerator type. + image_uri: Docker image URI to use for the custom job. + args: Arguments to pass to the Docker image. + accelerator_type: Type of accelerator. By default no accelerator is + requested. + accelerator_count: Number of accelerators. By default no accelerators are + requested. + encryption_spec_key_name: Customer-managed encryption key. If this is set, + then all resources created by the CustomJob will be encrypted with the + provided encryption key. Note that this is not supported for TPU at the + moment. + + Returns: + Custom job payload. + + Raises: + ValueError: if one of ``accelerator_count`` or ``accelerator_type`` is + specified, but the corresponding field is not valid. + """ + payload = { + 'display_name': display_name, + 'job_spec': { + 'worker_pool_specs': [{ + 'replica_count': '1', + 'machine_spec': {'machine_type': str(machine_type)}, + 'container_spec': {'image_uri': str(image_uri), 'args': args}, + }] + }, + } + + if accelerator_type and accelerator_count: + payload['job_spec']['worker_pool_specs'][0]['machine_spec'][ + 'accelerator_type' + ] = str(accelerator_type) + payload['job_spec']['worker_pool_specs'][0]['machine_spec'][ + 'accelerator_count' + ] = accelerator_count + elif accelerator_type and accelerator_count < 1: + raise ValueError( + 'Accelerator count must be at least 1 if accelerator type ' + f'is specified. Received accelerator_count == {accelerator_count}' + ) + elif accelerator_count and not accelerator_type: + raise ValueError( + 'Accelerator type must be specified if accelerator count is not 0.' + f'Received accelerator_type == {accelerator_type}.' + ) + + if encryption_spec_key_name: + payload['encryption_spec'] = {'kms_key_name': encryption_spec_key_name} + + return payload + + +def get_temp_location() -> str: + """Gets a task-specific location to store temporary files.""" + return os.path.join( + kfp.dsl.PIPELINE_ROOT_PLACEHOLDER, + kfp.dsl.PIPELINE_JOB_ID_PLACEHOLDER, + kfp.dsl.PIPELINE_TASK_ID_PLACEHOLDER, + 'temp', + ) + + +def get_default_image_uri(image_name: str) -> str: + """Gets the default image URI for a given image. + + The URI is resolved using environment variables that define the artifact + registry, image name modifications and tag. This method only works for images + that are not selected dynamically based on accelerator type. This is typically + true for CPU-only images. + + Args: + image_name: Name of the image to resolve. + + Returns: + URI of the image. + """ + return '/'.join([ + f'{env.PRIVATE_ARTIFACT_REGISTRY_LOCATION}-docker.pkg.dev', + env.PRIVATE_ARTIFACT_REGISTRY_PROJECT, + env.PRIVATE_ARTIFACT_REGISTRY, + f'{env.PRIVATE_IMAGE_NAME_PREFIX}{image_name}:{env.get_private_image_tag()}', + ]) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils_test.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils_test.py new file mode 100644 index 0000000000..a16c878fb3 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/llm/utils_test.py @@ -0,0 +1,62 @@ +"""Unit test for utils.""" + +from google_cloud_pipeline_components._implementation.llm import utils +import unittest + + +class UtilsTest(unittest.TestCase): + + def test_build_payload_basic_success(self): + machine_type = "n1-standard-1" + image_uri = "fake_image_uri" + args = ["--foo=bar"] + + expected_payload = { + "display_name": "test_with_encryption_spec_key_name", + "job_spec": { + "worker_pool_specs": [{ + "replica_count": "1", + "machine_spec": {"machine_type": machine_type}, + "container_spec": {"image_uri": image_uri, "args": args}, + }] + }, + } + + actual_payload = utils.build_payload( + display_name="test_with_encryption_spec_key_name", + machine_type=machine_type, + image_uri=image_uri, + args=args, + ) + self.assertDictEqual(expected_payload, actual_payload) + + def test_build_payload_with_encryption_spec_key_name(self): + machine_type = "n1-standard-1" + image_uri = "fake_image_uri" + args = ["--foo=bar"] + encryption_spec_key_name = "fake_cmek_key" + + expected_payload = { + "display_name": "test_with_encryption_spec_key_name", + "job_spec": { + "worker_pool_specs": [{ + "replica_count": "1", + "machine_spec": {"machine_type": machine_type}, + "container_spec": {"image_uri": image_uri, "args": args}, + }] + }, + "encryption_spec": {"kms_key_name": encryption_spec_key_name}, + } + + actual_payload = utils.build_payload( + display_name="test_with_encryption_spec_key_name", + machine_type=machine_type, + image_uri=image_uri, + args=args, + encryption_spec_key_name=encryption_spec_key_name, + ) + self.assertDictEqual(expected_payload, actual_payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/__init__.py index 11bc746456..07520b6f22 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/__init__.py @@ -15,22 +15,48 @@ import os +from google_cloud_pipeline_components._implementation.model_evaluation.chunking.component import chunking as ChunkingOp from google_cloud_pipeline_components._implementation.model_evaluation.data_sampler.component import evaluation_data_sampler as EvaluationDataSamplerOp from google_cloud_pipeline_components._implementation.model_evaluation.dataset_preprocessor.component import dataset_preprocessor_error_analysis as EvaluationDatasetPreprocessorOp +from google_cloud_pipeline_components._implementation.model_evaluation.endpoint_batch_predict.component import evaluation_llm_endpoint_batch_predict_pipeline_graph_component from google_cloud_pipeline_components._implementation.model_evaluation.error_analysis_annotation.component import error_analysis_annotation as ErrorAnalysisAnnotationOp from google_cloud_pipeline_components._implementation.model_evaluation.evaluated_annotation.component import evaluated_annotation as EvaluatedAnnotationOp +from google_cloud_pipeline_components._implementation.model_evaluation.feature_attribution.feature_attribution_component import feature_attribution as ModelEvaluationFeatureAttributionOp +from google_cloud_pipeline_components._implementation.model_evaluation.feature_attribution.feature_attribution_graph_component import feature_attribution_graph_component as FeatureAttributionGraphComponentOp from google_cloud_pipeline_components._implementation.model_evaluation.feature_extractor.component import feature_extractor_error_analysis as FeatureExtractorOp from google_cloud_pipeline_components._implementation.model_evaluation.import_evaluated_annotation.component import evaluated_annotation_import as ModelImportEvaluatedAnnotationOp from google_cloud_pipeline_components._implementation.model_evaluation.import_evaluation.component import model_evaluation_import as ModelImportEvaluationOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_classification_postprocessor.component import llm_classification_predictions_postprocessor_graph_component as LLMEvaluationClassificationPredictionsPostprocessorOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_embedding.evaluation_llm_embedding_pipeline import evaluation_llm_embedding_pipeline +from google_cloud_pipeline_components._implementation.model_evaluation.llm_embedding_retrieval.component import llm_embedding_retrieval as LLMEmbeddingRetrievalOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_evaluation.component import model_evaluation_text_generation as LLMEvaluationTextGenerationOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_information_retrieval_preprocessor.component import llm_information_retrieval_preprocessor as LLMInformationRetrievalPreprocessorOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_retrieval_metrics.component import llm_retrieval_metrics as LLMRetrievalMetricsOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_safety_bias.component import llm_safety_bias_metrics as LLMSafetyBiasMetricsOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_safety_bias.evaluation_llm_safety_bias_pipeline import evaluation_llm_safety_bias_pipeline from google_cloud_pipeline_components._implementation.model_evaluation.target_field_data_remover.component import target_field_data_remover as TargetFieldDataRemoverOp +from google_cloud_pipeline_components._implementation.model_evaluation.text2sql.evaluation_llm_text2sql_pipeline import evaluation_llm_text2sql_pipeline __all__ = [ + 'evaluation_llm_safety_bias_pipeline', + 'evaluation_llm_embedding_pipeline', + 'evaluation_llm_text2sql_pipeline', + 'evaluation_llm_endpoint_batch_predict_pipeline_graph_component', + 'ChunkingOp', 'EvaluationDataSamplerOp', 'EvaluationDatasetPreprocessorOp', 'ErrorAnalysisAnnotationOp', 'EvaluatedAnnotationOp', + 'FeatureAttributionGraphComponentOp', 'FeatureExtractorOp', + 'LLMEmbeddingRetrievalOp', + 'LLMEvaluationClassificationPredictionsPostprocessorOp', + 'LLMEvaluationTextGenerationOp', + 'LLMInformationRetrievalPreprocessorOp', + 'LLMRetrievalMetricsOp', + 'LLMSafetyBiasMetricsOp', + 'ModelEvaluationFeatureAttributionOp', 'ModelImportEvaluatedAnnotationOp', 'ModelImportEvaluationOp', 'TargetFieldDataRemoverOp', diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/chunking/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/chunking/__init__.py new file mode 100644 index 0000000000..34890407db --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/chunking/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation Chunking Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/chunking/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/chunking/component.py new file mode 100644 index 0000000000..9e3cda2a4b --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/chunking/component.py @@ -0,0 +1,97 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Embedding eval chunking component used in KFP pipelines.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from kfp.dsl import container_component +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +_IMAGE_URI = 'us-docker.pkg.dev/vertex-evaluation/public/llm:v0.3' + + +@container_component +def chunking( + gcp_resources: OutputPath(str), + project: str, + location: str, + input_text_gcs_dir: str, + output_bq_destination: str, + output_text_gcs_dir: str, + generation_threshold_microseconds: str, + display_name: str = 'chunking', + machine_type: str = 'n1-standard-8', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +): + """Chunk files in GCS and write to BigQuery table. + + Args: + project: The GCP project that runs the pipeline component. + location: The GCP region that runs the pipeline component. + input_text_gcs_dir: the GCS directory containing the files to chunk. DO NOT + include '/' at the end of the path. + output_bq_destination: The BigQuery table URI where the component will write + chunks to. + output_text_gcs_dir: The GCS folder to hold intermediate data. + generation_threshold_microseconds: only files created on/after this + generation threshold will be processed, in microseconds. + display_name: The name of the chunking job/component. + machine_type: The machine type of this custom job. + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. + network: The full name of the Compute Engine network to which the job should + be peered. For example, projects/12345/global/networks/myVPC. Format is of + the form projects/{project}/global/networks/{network}. Where {project} is + a project number, as in 12345, and {network} is a network name. Private + services access must already be configured for the network. If left + unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the + customJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources (str): + Serialized gcp_resources proto tracking the custom job. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=_IMAGE_URI, + args=[ + f'--process_chunking={True}', + f'--project={project}', + f'--location={location}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--input_text_gcs_dir={input_text_gcs_dir}', + f'--output_bq_destination={output_bq_destination}', + f'--output_text_gcs_dir={output_text_gcs_dir}', + f'--generation_threshold_microseconds={generation_threshold_microseconds}', + f'--gcp_resources={gcp_resources}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/data_sampler/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/data_sampler/component.py index 0ec8555d56..d974ff82c5 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/data_sampler/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/data_sampler/component.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG +from typing import List + +from google_cloud_pipeline_components._implementation.model_evaluation import version from kfp.dsl import container_component from kfp.dsl import ContainerSpec from kfp.dsl import OutputPath @@ -28,7 +30,7 @@ def evaluation_data_sampler( gcs_output_directory: OutputPath(list), project: str, location: str = 'us-central1', - gcs_source_uris: list = [], + gcs_source_uris: List[str] = [], bigquery_source_uri: str = '', instances_format: str = 'jsonl', sample_size: int = 10000, @@ -86,7 +88,7 @@ def evaluation_data_sampler( """ # fmt: on return ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/dataset_preprocessor/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/dataset_preprocessor/component.py index a4a47e28f5..50fc8a05a5 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/dataset_preprocessor/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/dataset_preprocessor/component.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + +from google_cloud_pipeline_components._implementation.model_evaluation import version from google_cloud_pipeline_components.types.artifact_types import VertexDataset -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG from kfp.dsl import container_component from kfp.dsl import ContainerSpec from kfp.dsl import IfPresentPlaceholder @@ -37,8 +39,8 @@ def dataset_preprocessor_error_analysis( test_dataset_annotation_set_name: str = '', training_dataset: Input[VertexDataset] = None, training_dataset_annotation_set_name: str = '', - test_dataset_storage_source_uris: list = [], - training_dataset_storage_source_uris: list = [], + test_dataset_storage_source_uris: List[str] = [], + training_dataset_storage_source_uris: List[str] = [], ): # fmt: off """Preprocesses datasets for Vision Error Analysis pipelines. @@ -107,7 +109,7 @@ def dataset_preprocessor_error_analysis( """ # fmt: on return ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=['python3', '/main.py'], args=[ '--task', diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/endpoint_batch_predict/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/endpoint_batch_predict/__init__.py new file mode 100644 index 0000000000..7edc3ee88e --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/endpoint_batch_predict/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation Endpoint Batch Predict Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/endpoint_batch_predict/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/endpoint_batch_predict/component.py new file mode 100644 index 0000000000..acb8048b9c --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/endpoint_batch_predict/component.py @@ -0,0 +1,229 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Endpoint batch predict component used in KFP pipelines.""" + +from typing import Dict, NamedTuple, Optional, Union +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from kfp import dsl +from kfp.dsl import container_component +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + +_IMAGE_URI = 'gcr.io/model-evaluation-dev/llm_eval:wjess-test' + + +@dsl.component +def add_json_escape_parameters(parameters: dict) -> str: + import json + + json_escaped_parameters = json.dumps(parameters).replace('"', '\\"') + return json_escaped_parameters + + +@container_component +def endpoint_batch_predict( + gcp_resources: OutputPath(str), + gcs_output_directory: OutputPath(str), + project: str, + location: str, + source_gcs_uri: str, + model_parameters: Optional[str] = None, + gcs_destination_output_uri_prefix: Optional[str] = '', + endpoint_id: Optional[str] = None, + publisher_model: Optional[str] = None, + qms_override: Optional[str] = None, + display_name: str = 'endpoint_batch_predict', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +): + """Returns the batch prediction results for a given batch of instances. + + Args: + project: Required. The GCP project that runs the pipeline component. + location: Required. The GCP region that runs the pipeline component. + source_gcs_uri: Google Cloud Storage URI to your instances to run + prediction on. The stored file format should be jsonl and each line + contains one Prediction instance. Instance should match Deployed model's + instance schema + gcs_destination_output_uri_prefix: The Google Cloud Storage location of + the directory where the output is to be written to. In the given + directory a new directory is created. Its name is + `prediction-model-`, where timestamp is in + YYYY-MM-DD-hh:mm:ss.sss format. Inside of it is file results.jsonl + endpoint_id: Required if no publisher_model is provided. The Endpoint ID + of the deployed the LLM to serve the prediction. When endpoint_id and + publisher_model are both provided, publisher_model will be used. + model_parameters: The parameters that govern the prediction. + publisher_model: Required if no endpoint_id is provided. Name of the + Publisher model. + location: Project the LLM Model is in. + qms_override: Manual control of a large language model's qms. Write up + when there's an approved quota increase for a LLM. Write down when + limiting qms of a LLM for this pipeline. Should be provided as a + dictionary, for example {'text-bison': 20}. For deployed model which + doesn't have google-vertex-llm-tuning-base-model-id label, override the + default here. + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted + to `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job + should be peered. For example, projects/12345/global/networks/myVPC. + Format is of the form projects/{project}/global/networks/{network}. + Where {project} is a project number, as in 12345, and {network} is a + network name. Private services access must already be configured for the + network. If left unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources (str): + Serialized gcp_resources proto tracking the custom job. + gcs_output_directory (str): + GCS directory where endpoint batch prediction results are stored. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=_IMAGE_URI, + args=[ + f'--endpoint_batch_predict={True}', + f'--project={project}', + f'--location={location}', + f'--source_gcs_uri={source_gcs_uri}', + f'--model_parameters={model_parameters}', + f'--gcs_destination_output_uri_prefix={gcs_destination_output_uri_prefix}', + f'--endpoint_id={endpoint_id}', + f'--publisher_model={publisher_model}', + f'--qms_override={qms_override}', + f'--gcs_output_directory={gcs_output_directory}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--gcp_resources={gcp_resources}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) + + +@dsl.pipeline(name='EvaludationLLMEndpointBatchPredictOp') +def evaluation_llm_endpoint_batch_predict_pipeline_graph_component( + project: str, + location: str, + source_gcs_uri: str, + model_parameters: Optional[Dict[str, Union[int, float]]] = {}, + gcs_destination_output_uri_prefix: Optional[str] = '', + endpoint_id: Optional[str] = None, + publisher_model: Optional[str] = None, + qms_override: Optional[str] = None, + display_name: str = 'endpoint_batch_predict', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +) -> NamedTuple('outputs', gcs_output_directory=str): + """The LLM Evaluation Text2SQL Pipeline. + + Args: + project: Required. The GCP project that runs the pipeline components. + location: Required. The GCP region that runs the pipeline components. + source_gcs_uri: Google Cloud Storage URI to your instances to run prediction + on. The stored file format should be jsonl and each line contains one + Prediction instance. Instance should match Deployed model's instance + schema + gcs_destination_output_uri_prefix: The Google Cloud Storage location of the + directory where the output is to be written to. In the given directory a + new directory is created. Its name is + `prediction-model-`, where timestamp is in + YYYY-MM-DD-hh:mm:ss.sss format. Inside of it is file results.jsonl + endpoint_id: Required if no publisher_model is provided. The Endpoint ID of + the deployed the LLM to serve the prediction. When endpoint_id and + publisher_model are both provided, publisher_model will be used. + model_parameters: The parameters that govern the prediction. + publisher_model: Required if no endpoint_id is provided. Name of the + Publisher model. + location: Project the LLM Model is in. + qms_override: Manual control of a large language model's qms. Write up when + there's an approved quota increase for a LLM. Write down when limiting qms + of a LLM for this pipeline. Should be provided as a dictionary, for + example {'text-bison': 20}. For deployed model which doesn't have + google-vertex-llm-tuning-base-model-id label, override the default here. + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should + be peered. For example, projects/12345/global/networks/myVPC. Format is of + the form projects/{project}/global/networks/{network}. Where {project} is + a project number, as in 12345, and {network} is a network name. Private + services access must already be configured for the network. If left + unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + NamedTuple: + gcs_output_directory (str): + GCS directory where endpoint batch prediction results are stored. + """ + outputs = NamedTuple('outputs', gcs_output_directory=str) + + endpoint_batch_predict_task = endpoint_batch_predict( + project=project, + location=location, + source_gcs_uri=source_gcs_uri, + model_parameters=add_json_escape_parameters( + parameters=model_parameters + ).output, + gcs_destination_output_uri_prefix=gcs_destination_output_uri_prefix, + endpoint_id=endpoint_id, + publisher_model=publisher_model, + qms_override=qms_override, + display_name=display_name, + machine_type=machine_type, + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ) + + return outputs( + gcs_output_directory=endpoint_batch_predict_task.outputs[ + 'gcs_output_directory' + ] + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/evaluated_annotation/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/evaluated_annotation/component.py index 8eccb13aa4..7dedf79c21 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/evaluated_annotation/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/evaluated_annotation/component.py @@ -15,7 +15,7 @@ from typing import Optional -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG +from google_cloud_pipeline_components._implementation.model_evaluation import version from kfp import dsl from kfp.dsl import Artifact from kfp.dsl import Input @@ -84,7 +84,7 @@ def evaluated_annotation( """ # fmt: on return dsl.ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=['python', '/main.py'], args=[ '--task', diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/__init__.py new file mode 100644 index 0000000000..67319b3f75 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation Feature Extractor Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/feature_attribution_component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/feature_attribution_component.py new file mode 100644 index 0000000000..e0f210f6ee --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/feature_attribution_component.py @@ -0,0 +1,179 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import version +from google_cloud_pipeline_components.types.artifact_types import BQTable +from kfp.dsl import Artifact +from kfp.dsl import ConcatPlaceholder +from kfp.dsl import container_component +from kfp.dsl import ContainerSpec +from kfp.dsl import IfPresentPlaceholder +from kfp.dsl import Input +from kfp.dsl import Metrics +from kfp.dsl import Output +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_JOB_ID_PLACEHOLDER +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER +from kfp.dsl import PIPELINE_TASK_ID_PLACEHOLDER + + +@container_component +def feature_attribution( + gcp_resources: OutputPath(str), + feature_attributions: Output[Metrics], + problem_type: str, + location: str = 'us-central1', + predictions_format: str = 'jsonl', + predictions_gcs_source: Input[Artifact] = None, + predictions_bigquery_source: Input[BQTable] = None, + dataflow_service_account: str = '', + dataflow_disk_size_gb: int = 50, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_workers_num: int = 1, + dataflow_max_workers_num: int = 5, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +): + # fmt: off + """Compute feature attribution on a trained model's batch explanation + results. + + Creates a dataflow job with Apache Beam and TFMA to compute feature + attributions. Will compute feature attribution for every target label if + possible, typically possible for AutoML Classification models. + + Args: + location: Location running feature attribution. If not + set, defaulted to `us-central1`. + problem_type: Problem type of the pipeline: one of `classification`, + `regression` and `forecasting`. + predictions_format: The file format for the batch + prediction results. `jsonl`, `csv`, and `bigquery` are the allowed + formats, from Vertex Batch Prediction. If not set, defaulted to `jsonl`. + predictions_gcs_source: An artifact with its + URI pointing toward a GCS directory with prediction or explanation files + to be used for this evaluation. For prediction results, the files should + be named "prediction.results-*" or "predictions_". For explanation + results, the files should be named "explanation.results-*". + predictions_bigquery_source: BigQuery table + with prediction or explanation data to be used for this evaluation. For + prediction results, the table column should be named "predicted_*". + dataflow_service_account: Service account to run the + dataflow job. If not set, dataflow will use the default worker service + account. For more details, see + https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine + executing the evaluation run. If not set, defaulted to `50`. + dataflow_machine_type: The machine type executing the + evaluation run. If not set, defaulted to `n1-standard-4`. + dataflow_workers_num: The number of workers executing the + evaluation run. If not set, defaulted to `10`. + dataflow_max_workers_num: The max number of workers + executing the evaluation run. If not set, defaulted to `25`. + dataflow_subnetwork: Dataflow's fully qualified subnetwork + name, when empty the default subnetwork will be used. More details: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow + workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key + for the Dataflow job. If this is set, then all resources created by the + Dataflow job will be encrypted with the provided encryption key. + force_runner_mode: Flag to choose Beam runner. Valid options are `DirectRunner` + and `Dataflow`. + project: Project to run feature attribution container. Defaults to the project in which the PipelineJob is run. + + Returns: + gcs_output_directory: JsonArray of the downsampled dataset GCS + output. + bigquery_output_table: String of the downsampled dataset BigQuery + output. + gcp_resources: Serialized gcp_resources proto tracking the dataflow + job. For more details, see + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + # fmt: on + return ContainerSpec( + image=version.EVAL_IMAGE_TAG, + command=[ + 'python3', + '/main.py', + ], + args=[ + '--task', + 'explanation', + '--setup_file', + '/setup.py', + '--project_id', + project, + '--location', + location, + '--problem_type', + problem_type, + '--root_dir', + f'{PIPELINE_ROOT_PLACEHOLDER}/{PIPELINE_JOB_ID_PLACEHOLDER}-{PIPELINE_TASK_ID_PLACEHOLDER}', + '--batch_prediction_format', + predictions_format, + IfPresentPlaceholder( + input_name='predictions_gcs_source', + then=[ + '--batch_prediction_gcs_source', + predictions_gcs_source.uri, + ], + ), + IfPresentPlaceholder( + input_name='predictions_bigquery_source', + then=[ + '--batch_prediction_bigquery_source', + ConcatPlaceholder([ + 'bq://', + predictions_bigquery_source.metadata['projectId'], + '.', + predictions_bigquery_source.metadata['datasetId'], + '.', + predictions_bigquery_source.metadata['tableId'], + ]), + ], + ), + '--dataflow_job_prefix', + f'evaluation-feautre-attribution-{PIPELINE_JOB_ID_PLACEHOLDER}-{PIPELINE_TASK_ID_PLACEHOLDER}', + '--dataflow_service_account', + dataflow_service_account, + '--dataflow_disk_size', + dataflow_disk_size_gb, + '--dataflow_machine_type', + dataflow_machine_type, + '--dataflow_workers_num', + dataflow_workers_num, + '--dataflow_max_workers_num', + dataflow_max_workers_num, + '--dataflow_subnetwork', + dataflow_subnetwork, + '--dataflow_use_public_ips', + dataflow_use_public_ips, + '--kms_key_name', + encryption_spec_key_name, + '--force_runner_mode', + force_runner_mode, + '--gcs_output_path', + feature_attributions.path, + '--gcp_resources', + gcp_resources, + '--executor_input', + '{{$}}', + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/feature_attribution_graph_component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/feature_attribution_graph_component.py new file mode 100644 index 0000000000..63d827946d --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/feature_attribution/feature_attribution_graph_component.py @@ -0,0 +1,245 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Graph Component for feature attribution evaluation.""" + +from typing import List, NamedTuple + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation.data_sampler.component import evaluation_data_sampler as EvaluationDataSamplerOp +from google_cloud_pipeline_components._implementation.model_evaluation.feature_attribution.feature_attribution_component import feature_attribution as ModelEvaluationFeatureAttributionOp +from google_cloud_pipeline_components.types.artifact_types import VertexModel +from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp +import kfp + + +@kfp.dsl.pipeline(name='feature-attribution-graph-component') +def feature_attribution_graph_component( # pylint: disable=dangerous-default-value + location: str, + prediction_type: str, + vertex_model: VertexModel, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_explanation_metadata: dict = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: dict = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_data_sample_size: int = 10000, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +) -> NamedTuple('outputs', feature_attributions=kfp.dsl.Metrics): + """A pipeline to compute feature attributions by sampling data for batch explanations. + + This pipeline guarantees support for AutoML Tabular models that contain a + valid explanation_spec. + + Args: + location: The GCP region that runs the pipeline components. + prediction_type: The type of prediction the model is to produce. + "classification", "regression", or "forecasting". + vertex_model: The Vertex model artifact used for batch explanation. + batch_predict_instances_format: The format in which instances are given, + must be one of the Model's supportedInputStorageFormats. For more details + about this input config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location + of the directory where the output is to be written to. In the given + directory a new directory is created. Its name is + `prediction--`, where timestamp is in + YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + `predictions_0001.`, `predictions_0002.`, ..., + `predictions_N.` are created where `` depends on + chosen `predictions_format`, and N may equal 0001 and depends on the total + number of successfully predicted instances. If the Model has both + `instance` and `prediction` schemata defined then each such file contains + predictions as per the `predictions_format`. If prediction for any + instance failed (partially or completely), then an additional + `errors_0001.`, `errors_0002.`,..., + `errors_N.` files are created (N depends on total number of + failed predictions). These files contain the failed instances, as per + their schema, followed by an additional `error` field which as value has + `google.rpc.Status` containing only `code` and `message` fields. For more + details about this output config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your + instances to run batch prediction on. May contain wildcards. For more + information on wildcards, see + https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For + more details about this input config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to + run batch prediction on. May contain wildcards. For more details about + this input config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the + predictions. Must be one of the Model's supportedOutputStorageFormats. For + more details about this output config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location + where the output is to be written to. In the given project a new dataset + is created with name `prediction__` + where is made BigQuery-dataset-name compatible (for example, most special + characters become underscores), and timestamp is in + YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two + tables will be created, `predictions`, and `errors`. If the Model has both + `instance` and `prediction` schemata defined then the tables have columns + as follows: The `predictions` table contains instances for which the + prediction succeeded, it has columns as per a concatenation of the Model's + instance and prediction schemata. The `errors` table contains rows for + which the prediction has failed, it has instance columns, as per the + instance schema, followed by a single "errors" column, which as values has + `google.rpc.Status` represented as a STRUCT, and containing only `code` + and `message`. For more details about this output config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction + on dedicated resources. If the Model supports DEDICATED_RESOURCES this + config may be provided (and the job will use these resources). If the + Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. + For more details about the BatchDedicatedResources, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + For more details about the machine spec, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at + the start of the batch operation. If not set, Vertex AI decides starting + number, not greater than `max_replica_count`. Only used if `machine_type` + is set. + batch_predict_max_replica_count: The maximum number of machine replicas the + batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for + this BatchPredictionJob. Can be specified only if `generate_explanation` + is set to `True`. This value overrides the value of + `Model.explanation_metadata`. All fields of `explanation_metadata` are + optional in the request. If a field of the `explanation_metadata` object + is not populated, the corresponding field of the + `Model.explanation_metadata` object is inherited. For more details, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for + Model's predictions. Can be specified only if `generate_explanation` is + set to `True`. This value overrides the value of + `Model.explanation_parameters`. All fields of `explanation_parameters` are + optional in the request. If a field of the `explanation_parameters` object + is not populated, the corresponding field of the + `Model.explanation_parameters` object is inherited. For more details, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the + input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be + attached to the machine as per `batch_predict_accelerator_count`. Only + used if `batch_predict_machine_type` is set. For more details about the + machine spec, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the + `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is + set. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation + components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation + components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, + resources created by this pipeline will be encrypted with the provided + encryption key. Has the form: + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + The key needs to be in the same region as where the compute resource is + created. + force_runner_mode: Indicate the runner mode to use forcely. Valid options + are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the + project in which the PipelineJob is run. + + Returns: + A system.Metrics artifact with feature attributions. + """ + outputs = NamedTuple('outputs', feature_attributions=kfp.dsl.Metrics) + + # Sample the input dataset for a quicker batch explanation. + data_sampler_task = EvaluationDataSamplerOp( + project=project, + location=location, + gcs_source_uris=batch_predict_gcs_source_uris, + bigquery_source_uri=batch_predict_bigquery_source_uri, + instances_format=batch_predict_instances_format, + sample_size=batch_predict_explanation_data_sample_size, + force_runner_mode=force_runner_mode, + ) + + # Run batch explain. + batch_explain_task = ModelBatchPredictOp( + project=project, + location=location, + model=vertex_model, + job_display_name='model-registry-batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=data_sampler_task.outputs['gcs_output_directory'], + bigquery_source_input_uri=data_sampler_task.outputs[ + 'bigquery_output_table' + ], + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, + bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + generate_explanation=True, + explanation_parameters=batch_predict_explanation_parameters, + explanation_metadata=batch_predict_explanation_metadata, + machine_type=batch_predict_machine_type, + starting_replica_count=batch_predict_starting_replica_count, + max_replica_count=batch_predict_max_replica_count, + encryption_spec_key_name=encryption_spec_key_name, + accelerator_type=batch_predict_accelerator_type, + accelerator_count=batch_predict_accelerator_count, + ) + + # Generate feature attributions from explanations. + feature_attribution_task = ModelEvaluationFeatureAttributionOp( + project=project, + location=location, + problem_type=prediction_type, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_explain_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_explain_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + ) + + return outputs( + feature_attributions=feature_attribution_task.outputs[ + 'feature_attributions' + ] + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py index e33c75058b..2d4976d1ed 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/import_evaluation/component.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import Optional +from typing import List, Optional from google_cloud_pipeline_components import _image from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics @@ -29,6 +29,7 @@ def model_evaluation_import( model: Input[VertexModel], gcp_resources: dsl.OutputPath(str), + evaluation_resource_name: dsl.OutputPath(str), metrics: Optional[Input[Metrics]] = None, problem_type: Optional[str] = None, classification_metrics: Optional[Input[ClassificationMetrics]] = None, @@ -39,10 +40,11 @@ def model_evaluation_import( summarization_metrics: Optional[Input[Metrics]] = None, explanation: Optional[Input[Metrics]] = None, feature_attributions: Optional[Input[Metrics]] = None, - display_name: Optional[str] = "", - dataset_path: Optional[str] = "", - dataset_paths: Optional[list] = [], - dataset_type: Optional[str] = "", + embedding_metrics: Optional[Input[Metrics]] = None, + display_name: str = "", + dataset_path: str = "", + dataset_paths: List[str] = [], + dataset_type: str = "", ): # fmt: off """Imports a model evaluation artifact to an existing Vertex model with @@ -58,19 +60,31 @@ def model_evaluation_import( uploaded evaluation. metrics: Path of metrics generated from an evaluation component. problem_type: The problem type of the metrics being imported to the - VertexModel. `classification`, `regression`, and `forecasting` are the + VertexModel. `classification`, `regression`, `forecasting`, + `text-generation`, `question-answering`, and `summarization` are the currently supported problem types. Must be provided when `metrics` is provided. - classification_metrics: Path of classification metrics generated from the - classification evaluation component. - forecasting_metrics: Path of forecasting metrics generated from the - forecasting evaluation component. - regression_metrics: Path of regression metrics generated from the regression - evaluation component. + classification_metrics: google.ClassificationMetrics artifact generated from + the ModelEvaluationClassificationOp component. + forecasting_metrics: google.ForecastingMetrics artifact generated from + the ModelEvaluationForecastingOp component. + regression_metrics: google.ClassificationMetrics artifact generated from + the ModelEvaluationRegressionOp component. + text_generation_metrics: system.Metrics artifact generated from + the LLMEvaluationTextGenerationOp component. Subject to change to + google.TextGenerationMetrics. + question_answering_metrics: system.Metrics artifact generated from + the LLMEvaluationTextGenerationOp component. Subject to change to + google.QuestionAnsweringMetrics. + summarization_metrics: system.Metrics artifact generated from + the LLMEvaluationTextGenerationOp component. Subject to change to + google.SummarizationMetrics. explanation: Path for model explanation metrics generated from an evaluation component. feature_attributions: The feature attributions metrics artifact generated from the feature attribution component. + embedding_metrics: The embedding metrics artifact generated from the + embedding retrieval metrics component. display_name: The display name for the uploaded model evaluation resource. """ # fmt: on @@ -138,7 +152,7 @@ def model_evaluation_import( input_name="summarization_metrics", then=[ "--summarization_metrics", - "{{$.inputs.artifacts['summarization_metrics'].uri}}", + summarization_metrics.uri, ], ), dsl.IfPresentPlaceholder( @@ -148,6 +162,13 @@ def model_evaluation_import( feature_attributions.uri, ], ), + dsl.IfPresentPlaceholder( + input_name="embedding_metrics", + then=[ + "--embedding_metrics", + embedding_metrics.uri, + ], + ), dsl.IfPresentPlaceholder( input_name="problem_type", then=[ @@ -171,5 +192,7 @@ def model_evaluation_import( model.metadata["resourceName"], "--gcp_resources", gcp_resources, + "--evaluation_resource_name", + evaluation_resource_name, ], ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/__init__.py new file mode 100644 index 0000000000..522a3bf396 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Evaluation Predictions Postprocessor Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/component.py new file mode 100644 index 0000000000..24cfa5da95 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_classification_postprocessor/component.py @@ -0,0 +1,244 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LLM Classification Postprocessor component.""" + +from typing import List, NamedTuple, Optional + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output +from kfp.dsl import OutputPath + + +@dsl.component +def add_json_escape_class_labels(class_labels: list) -> str: + import json + + json_escaped_class_labels = json.dumps(class_labels).replace('"', '\\"') + return json_escaped_class_labels + + +@dsl.container_component +def llm_classification_predictions_postprocessor_internal( + gcp_resources: OutputPath(str), + postprocessed_predictions_gcs_source: Output[Artifact], + postprocessed_class_labels: OutputPath(list), + project: str, + location: str, + batch_prediction_results: Input[Artifact], + class_labels: str, + display_name: str = 'llm-classification-predictions-postprocessor', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + reserved_ip_ranges: Optional[List[str]] = None, + encryption_spec_key_name: str = '', +): + """Postprocesses LLM predictions for evaluating classification task. + + For each output string, find the first appearance of a class label in the + list of classes, and output the index of this class in a one-hot encoding + format for evaluation. If the output string does not contain any class labels + from the list, label it as “UNKNOWN”. + + Constraints + 1. In rare cases, if the model outputs verbose answers like "The topic of + the text is not business, but is health". In this case, the first answer in + the list the model outputs isn't what the model actually chose, and the + postprocessor output would be incorrect. + 2. Cannot handle cases where class names are substrings of each other. For + example, "toxic, nontoxic". + + Args: + project: The GCP project that runs the pipeline component. + location: The GCP region that runs the pipeline component. + batch_prediction_results: An Artifact pointing toward a GCS directory with + prediction files to be used for this component. + class_labels: String that is JSON array escapedclass names for the + target_field, in the same order they appear in the batch predictions input + file. + display_name: The name of the custom job. + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. + network: The full name of the Compute Engine network to which the job should + be peered. For example, `projects/12345/global/networks/myVPC`. Format is + of the form `projects/{project}/global/networks/{network}`. Where + `{project}` is a project number, as in `12345`, and `{network}` is a + network name, as in `myVPC`. To specify this field, you must have already + configured VPC Network Peering for Vertex AI + (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left + unspecified, the job is not peered with any network. + reserved_ip_ranges: A list of names for the reserved ip ranges under the VPC + network that can be used for this job. If set, we will deploy the job + within the provided ip ranges. Otherwise, the job will be deployed to any + ip ranges under the provided VPC network. + encryption_spec_key_name: Customer-managed encryption key options. If set, + resources created by this Custom Job will be encrypted with the provided + encryption key. Has the form: + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + The key needs to be in the same region as where the compute resource is + created. + + Returns: + postprocessed_predictions_gcs_source: A string URI pointing toward a GCS + directory with postprocessed prediction files to be used for Evaluation + component. + postprocessed_class_labels: The list of class names for the target_field + with an additional field named "UNKNOWN", in the same order they appear in + the batch predictions input file. + gcp_resources: Serialized gcp_resources proto tracking the custom job. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + '--postprocessor', + 'true', + '--batch_prediction_results', + batch_prediction_results.path, + '--postprocessed_predictions_gcs_source', + postprocessed_predictions_gcs_source.path, + '--class_labels', + class_labels, + '--postprocessed_class_labels', + postprocessed_class_labels, + '--executor_input', + '{{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + reserved_ip_ranges=reserved_ip_ranges, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) + + +@dsl.pipeline(name='ModelEvaluationLLMClassificationPredictionsPostprocessorOp') +def llm_classification_predictions_postprocessor_graph_component( + project: str, + location: str, + batch_prediction_results: Input[Artifact], + class_labels: List[str], + display_name: str = 'llm-classification-predictions-postprocessor', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + reserved_ip_ranges: Optional[List[str]] = None, + encryption_spec_key_name: str = '', +) -> NamedTuple( + 'outputs', + postprocessed_predictions_gcs_source=Artifact, + postprocessed_class_labels=List[str], +): + """Graph component to postprocess LLM predictions for evaluating classification task. + + For each output string, find the first appearance of a class label in the + list of classes, and output the index of this class in a one-hot encoding + format for evaluation. If the output string does not contain any class labels + from the list, label it as “UNKNOWN”. + + Constraints + 1. In rare cases, if the model outputs verbose answers like "The topic of + the text is not business, but is health". In this case, the first answer in + the list the model outputs isn't what the model actually chose, and the + postprocessor output would be incorrect. + 2. Cannot handle cases where class names are substrings of each other. For + example, "toxic, nontoxic". + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + batch_prediction_results: An Artifact pointing toward a GCS directory with + prediction files to be used for this component. + class_labels: The JSON array of class names for the target_field, in the + same order they appear in the batch predictions input file. + display_name: The name of the custom job. + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. + network: The full name of the Compute Engine network to which the job should + be peered. For example, `projects/12345/global/networks/myVPC`. Format is + of the form `projects/{project}/global/networks/{network}`. Where + `{project}` is a project number, as in `12345`, and `{network}` is a + network name, as in `myVPC`. To specify this field, you must have already + configured VPC Network Peering for Vertex AI + (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left + unspecified, the job is not peered with any network. + reserved_ip_ranges: A list of names for the reserved ip ranges under the VPC + network that can be used for this job. If set, we will deploy the job + within the provided ip ranges. Otherwise, the job will be deployed to any + ip ranges under the provided VPC network. + encryption_spec_key_name: Customer-managed encryption key options. If set, + resources created by this Custom Job will be encrypted with the provided + encryption key. Has the form: + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + The key needs to be in the same region as where the compute resource is + created. + + Returns: + NamedTuple: + postprocessed_predictions_gcs_source: A string URI pointing toward a GCS + directory with postprocessed prediction files to be used for Evaluation + component. + postprocessed_class_labels: The list of class names for the target_field + with an additional field named "UNKNOWN", in the same order they appear + in the batch predictions input file. + """ + outputs = NamedTuple( + 'outputs', + postprocessed_predictions_gcs_source=Artifact, + postprocessed_class_labels=List[str], + ) + + postprocessor_task = llm_classification_predictions_postprocessor_internal( + project=project, + batch_prediction_results=batch_prediction_results, + class_labels=add_json_escape_class_labels( + class_labels=class_labels + ).output, + location=location, + display_name=display_name, + machine_type=machine_type, + service_account=service_account, + network=network, + reserved_ip_ranges=reserved_ip_ranges, + encryption_spec_key_name=encryption_spec_key_name, + ) + + return outputs( + postprocessed_predictions_gcs_source=postprocessor_task.outputs[ + 'postprocessed_predictions_gcs_source' + ], + postprocessed_class_labels=postprocessor_task.outputs[ + 'postprocessed_class_labels' + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/__init__.py new file mode 100644 index 0000000000..5a34bea2c6 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Embedding Evaluation Pipeline.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/evaluation_llm_embedding_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/evaluation_llm_embedding_pipeline.py new file mode 100644 index 0000000000..f4cbced09f --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding/evaluation_llm_embedding_pipeline.py @@ -0,0 +1,248 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LLM embedding evaluation pipeline based on information retrieval (IR) task.""" + +from google_cloud_pipeline_components._implementation.model_evaluation.import_evaluation.component import model_evaluation_import as ModelImportEvaluationOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_embedding_retrieval.component import llm_embedding_retrieval as LLMEmbeddingRetrievalOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_information_retrieval_preprocessor.component import llm_information_retrieval_preprocessor as LLMInformationRetrievalPreprocessorOp +from google_cloud_pipeline_components._implementation.model_evaluation.llm_retrieval_metrics.component import llm_retrieval_metrics as LLMRetrievalMetricsOp +from google_cloud_pipeline_components.types.artifact_types import VertexModel +from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp +import kfp +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +_PIPELINE_NAME = 'evaluation-llm-embedding-pipeline' + + +@kfp.dsl.pipeline(name=_PIPELINE_NAME) +def evaluation_llm_embedding_pipeline( + project: str, + location: str, + corpus_gcs_source: str, + query_gcs_source: str, + golden_docs_gcs_source: str, + model_name: str, + embedding_chunking_function: str = 'langchain-RecursiveCharacterTextSplitter', + embedding_chunk_size: int = 0, + embedding_chunk_overlap: int = 0, + embedding_retrieval_combination_function: str = 'max', + batch_predict_instances_format: str = 'jsonl', + batch_predict_predictions_format: str = 'jsonl', + embedding_retrieval_top_n: int = 10, + retrieval_metrics_top_k_list: str = '10', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + runner: str = 'DirectRunner', + dataflow_service_account: str = '', + dataflow_disk_size_gb: int = 50, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_workers_num: int = 1, + dataflow_max_workers_num: int = 5, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +): + """The LLM Embedding Evaluation Pipeline. + + Args: + project: Required. The GCP project that runs the pipeline components. + location: Required. The GCP region that runs the pipeline components. + corpus_gcs_source: The gcs location for json file containing corpus + documents. + query_gcs_source: The gcs location for json file containing query documents. + golden_docs_gcs_source: The gcs location for csv file containing mapping of + each query to the golden docs. + model_name: The path for model to generate embeddings. + embedding_chunking_function: function used to split a document into chunks. + Supported values are `langchain-RecursiveCharacterTextSplitter` and + `sentence-splitter`. langchain-RecursiveCharacterTextSplitter: + langchain.text_splitter.RecursiveCharacterTextSplitter, with configurable + chunk_size and chunk_overlap. sentence-splitter: splitter that will not + break in the middle of a sentence. embedding_chunk_size and + embedding_chunk_overlap are measured by the number of tokens. + embedding_chunk_size: The length of each document chunk. If 0, chunking is + not enabled. + embedding_chunk_overlap: The length of the overlap part between adjacent + chunks. Will only be used if embedding_chunk_size > 0. + embedding_retrieval_combination_function: The function to combine + query-chunk similarities to query-doc similarity. Supported functions are + avg, max, and sum. + batch_predict_instances_format: The format in which instances are given, + must be one of the Model's supportedInputStorageFormats. If not set, + default to "jsonl". For more details about this input config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_instances_format: The format in which perdictions are made, + must be one of the Model's supportedInputStorageFormats. If not set, + default to "jsonl". For more details about this input config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + embedding_retrieval_top_n: Top N docs will be retrieved for each query, + based on similarity. + retrieval_metrics_top_k_list: k values for retrieval metrics, for example, + precision@k, accuracy@k, etc. If more than one value, separated by comma. + e.g., "1,5,10". + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Service account to run the dataflow job. If not set, + dataflow will use the default worker service account. For more details, + see + https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + network: Dataflow's fully qualified subnetwork name, when empty the default + subnetwork will be used. More details: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + runner: runner for the beam pipeline. DirectRunner and DataflowRunner are + supported. + dataflow_service_account: Service account to run the dataflow job. If not + set, dataflow will use the default worker service account. For more + details, see + https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the + evaluation run. + dataflow_machine_type: The machine type executing the evaluation run. + dataflow_workers_num: The number of workers executing the evaluation run. + dataflow_max_workers_num: The max number of workers executing the evaluation + run. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. More details: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + """ + + preprocessing_task = LLMInformationRetrievalPreprocessorOp( + project=project, + location=location, + corpus_gcs_source=corpus_gcs_source, + query_gcs_source=query_gcs_source, + golden_docs_gcs_source=golden_docs_gcs_source, + machine_type=machine_type, + service_account=service_account, + network=network, + runner=runner, + embedding_chunking_function=embedding_chunking_function, + embedding_chunk_size=embedding_chunk_size, + embedding_chunk_overlap=embedding_chunk_overlap, + dataflow_service_account=dataflow_service_account, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_machine_type=dataflow_machine_type, + dataflow_workers_num=dataflow_workers_num, + dataflow_max_workers_num=dataflow_max_workers_num, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + ) + + get_vertex_model_task = kfp.dsl.importer( + artifact_uri=( + f'https://{location}-aiplatform.googleapis.com/v1/{model_name}' + ), + artifact_class=VertexModel, + metadata={'resourceName': model_name}, + ) + get_vertex_model_task.set_display_name('get-vertex-model') + + batch_predict_corpus = ModelBatchPredictOp( + project=project, + location=location, + model=get_vertex_model_task.outputs['artifact'], + job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=preprocessing_task.outputs[ + 'predictions_corpus_gcs_source' + ], + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=( + f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_output' + ), + encryption_spec_key_name=encryption_spec_key_name, + ) + + batch_predict_query = ModelBatchPredictOp( + project=project, + location=location, + model=get_vertex_model_task.outputs['artifact'], + job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=preprocessing_task.outputs[ + 'predictions_query_gcs_source' + ], + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=( + f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_output' + ), + encryption_spec_key_name=encryption_spec_key_name, + ) + + # TODO(b/290838262): Revisit if/when the concurrent jobs limit is increased/removed. + batch_predict_query.after(batch_predict_corpus) + + embedding_retrieval_task = LLMEmbeddingRetrievalOp( + project=project, + location=location, + query_embedding_source_directory=batch_predict_query.outputs[ + 'gcs_output_directory' + ], + doc_embedding_source_directory=batch_predict_corpus.outputs[ + 'gcs_output_directory' + ], + embedding_retrieval_top_n=embedding_retrieval_top_n, + embedding_retrieval_combination_function=embedding_retrieval_combination_function, + machine_type=machine_type, + service_account=service_account, + network=network, + runner=runner, + dataflow_service_account=dataflow_service_account, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_machine_type=dataflow_machine_type, + dataflow_workers_num=dataflow_workers_num, + dataflow_max_workers_num=dataflow_max_workers_num, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + ) + + retrieval_metrics_task = LLMRetrievalMetricsOp( + project=project, + location=location, + golden_docs_pattern=preprocessing_task.outputs[ + 'embedding_retrieval_gcs_source' + ], + embedding_retrieval_results_pattern=embedding_retrieval_task.outputs[ + 'embedding_retrieval_results_path' + ], + retrieval_metrics_top_k_list=retrieval_metrics_top_k_list, + machine_type=machine_type, + service_account=service_account, + network=network, + runner=runner, + dataflow_service_account=dataflow_service_account, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_machine_type=dataflow_machine_type, + dataflow_workers_num=dataflow_workers_num, + dataflow_max_workers_num=dataflow_max_workers_num, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + ) + + ModelImportEvaluationOp( + embedding_metrics=retrieval_metrics_task.outputs['retrieval_metrics'], + model=get_vertex_model_task.outputs['artifact'], + display_name=_PIPELINE_NAME, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/__init__.py new file mode 100644 index 0000000000..c6e8df70e4 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Embedding Evaluation Retrieval Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/component.py new file mode 100644 index 0000000000..4f481e5b4b --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_embedding_retrieval/component.py @@ -0,0 +1,142 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python LLM Embedding Retrieval component used in KFP pipelines.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp.dsl import Artifact +from kfp.dsl import container_component +from kfp.dsl import Input +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +@container_component +def llm_embedding_retrieval( + gcp_resources: OutputPath(str), + embedding_retrieval_results_path: OutputPath(str), + project: str, + location: str, + query_embedding_source_directory: Input[Artifact], + doc_embedding_source_directory: Input[Artifact], + embedding_retrieval_top_n: int, + embedding_retrieval_combination_function: str = 'max', + display_name: str = 'llm_embedding_retrieval_component', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + runner: str = 'DirectRunner', + dataflow_service_account: str = '', + dataflow_disk_size_gb: int = 50, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_workers_num: int = 1, + dataflow_max_workers_num: int = 5, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +): + """Top N doc retrieval for queries, based on their embedding similarities. + + Args: + project: Required. The GCP project that runs the pipeline component. + location: Required. The GCP region that runs the pipeline component. + query_embedding_source_directory: Required. Directory where query + embedding results are saved. + doc_embedding_source_directory: Required. Directory where doc embedding + results are saved. + embedding_retrieval_top_n: Required. Top N docs will be retrieved for each + query, based on similarity. + embedding_retrieval_combination_function: The function to combine + query-chunk similarities to query-doc similarity. Supported functions + are avg, max, and sum. + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted + to `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job + should be peered. For example, projects/12345/global/networks/myVPC. + Format is of the form projects/{project}/global/networks/{network}. + Where {project} is a project number, as in 12345, and {network} is a + network name. Private services access must already be configured for the + network. If left unspecified, the job is not peered with any network. + runner: runner for the beam pipeline. DirectRunner and DataflowRunner are + supported. + dataflow_service_account: Service account to run the dataflow job. If not + set, dataflow will use the default worker service account. For more + details, see + https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the + evaluation run. + dataflow_machine_type: The machine type executing the evaluation run. + dataflow_workers_num: The number of workers executing the evaluation run. + dataflow_max_workers_num: The max number of workers executing the + evaluation run. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when + empty the default subnetwork will be used. More details: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources (str): + Serialized gcp_resources proto tracking the custom job. + embedding_retrieval_results_path (str): + The prefix of sharded GCS output of document retrieval results based on + embedding similarity, in JSONL format. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + f'--embedding_retrieval={True}', + f'--project={project}', + f'--location={location}', + f'--query_embedding_source_directory={query_embedding_source_directory.path}', + f'--doc_embedding_source_directory={doc_embedding_source_directory.path}', + f'--embedding_retrieval_top_n={embedding_retrieval_top_n}', + f'--embedding_retrieval_combination_function={embedding_retrieval_combination_function}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--gcp_resources={gcp_resources}', + f'--embedding_retrieval_results_path={embedding_retrieval_results_path}', + f'--runner={runner}', + f'--dataflow_service_account={dataflow_service_account}', + f'--dataflow_disk_size={dataflow_disk_size_gb}', + f'--dataflow_machine_type={dataflow_machine_type}', + f'--dataflow_workers_num={dataflow_workers_num}', + f'--dataflow_max_workers_num={dataflow_max_workers_num}', + f'--dataflow_subnetwork={dataflow_subnetwork}', + f'--dataflow_use_public_ips={dataflow_use_public_ips}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/__init__.py new file mode 100644 index 0000000000..07f468a2ae --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Evaluation Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py new file mode 100644 index 0000000000..972e752097 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_evaluation/component.py @@ -0,0 +1,139 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Text Generation LLM Evaluation component.""" + +from typing import List + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Metrics +from kfp.dsl import Output +from kfp.dsl import OutputPath + + +@dsl.container_component +def model_evaluation_text_generation( + gcp_resources: OutputPath(str), + evaluation_metrics: Output[Metrics], + project: str, + location: str, + evaluation_task: str = 'text-generation', + target_field_name: str = 'instance.ground_truth', + prediction_field_name: str = 'predictions.content', + predictions_format: str = 'jsonl', + joined_predictions_gcs_source: dsl.Input[Artifact] = None, + predictions_gcs_source: dsl.Input[Artifact] = None, + ground_truth_gcs_source: str = '', + display_name: str = 'model-evaluation-text-generation', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + reserved_ip_ranges: List[str] = [], + encryption_spec_key_name: str = '', +): + """Computes evaluation metrics of a text generation model. + + Supports evaluating large language models performing the following generative + tasks: `summarization`, `question-answering`, and `text-generation`. + + Args: + project: The GCP project that runs the pipeline component. + location: The GCP region that runs the pipeline component. + evaluation_task: The task that the large language model will be evaluated + on. The evaluation component computes a set of metrics relevant to that + specific task. Currently supported tasks are: `summarization`, + `question-answering`, and `text-generation`. + target_field_name: The full name path of the features target field in the + predictions file. Formatted to be able to find nested columns, delimited + by `.`. Alternatively referred to as the ground truth (or + ground_truth_column) field. If not set, defaulted to + `inputs.ground_truth`. + prediction_field_name: The full name path of the prediction field in the + prediction file. Formatted to be able to find nested columns, delimited by + `.`. If not set, defaulted to `predictions.content`. + predictions_format: The file format for the LLM Batch Prediction results. + `jsonl` is currently the only allowed format. If not set, defaulted to + `jsonl`. + joined_predictions_gcs_source: An Artifact with an URI pointing toward a GCS + directory or a GCS file with joined prediction & ground truth files to be + used for this evaluation. + predictions_gcs_source: An Artifact with an URI pointing toward a GCS + directory with only prediction files to be used for this evaluation. + ground_truth_gcs_source: A storage URI pointing toward a GCS directory with + only ground truth files to be used for this evaluation. + display_name: The name of the evaluation custom job. + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should + be peered. For example, `projects/12345/global/networks/myVPC`. Format is + of the form `projects/{project}/global/networks/{network}`. Where + `{project}` is a project number, as in `12345`, and `{network}` is a + network name, as in `myVPC`. To specify this field, you must have already + configured VPC Network Peering for Vertex AI + (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left + unspecified, the job is not peered with any network. + reserved_ip_ranges: A list of names for the reserved ip ranges under the VPC + network that can be used for this job. If set, we will deploy the job + within the provided ip ranges. Otherwise, the job will be deployed to any + ip ranges under the provided VPC network. + encryption_spec_key_name: Customer-managed encryption key options. If set, + resources created by this pipeline will be encrypted with the provided + encryption key. Has the form: + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + The key needs to be in the same region as where the compute resource is + created. + + Returns: + evaluation_metrics: `Metrics` artifact representing the language model + evaluation metrics. + gcp_resources: Serialized gcp_resources proto tracking the custom job. + For more details, see + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + f'--evaluation_task={evaluation_task}', + f'--target_field_name={target_field_name}', + f'--prediction_field_name={prediction_field_name}', + f'--predictions_format={predictions_format}', + f'--joined_predictions_gcs_source={joined_predictions_gcs_source.uri}', + f'--predictions_gcs_source={predictions_gcs_source.uri}', + f'--ground_truth_gcs_source={ground_truth_gcs_source}', + f'--evaluation_metrics_output_path={evaluation_metrics.path}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + reserved_ip_ranges=reserved_ip_ranges, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/__init__.py new file mode 100644 index 0000000000..ddb8ed570f --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Embedding Evaluation Preprocessing Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/component.py new file mode 100644 index 0000000000..ee9d3fecdc --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_information_retrieval_preprocessor/component.py @@ -0,0 +1,161 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Information Retrieval preprocessor component used in KFP pipelines.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp.dsl import container_component +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +@container_component +def llm_information_retrieval_preprocessor( + gcp_resources: OutputPath(str), + predictions_query_gcs_source: OutputPath(list), + predictions_corpus_gcs_source: OutputPath(list), + embedding_retrieval_gcs_source: OutputPath(str), + project: str, + location: str, + corpus_gcs_source: str, + query_gcs_source: str, + golden_docs_gcs_source: str, + embedding_chunking_function: str = 'langchain-RecursiveCharacterTextSplitter', + embedding_chunk_size: int = 0, + embedding_chunk_overlap: int = 0, + display_name: str = 'information-retrieval-preprocessor', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + runner: str = 'DirectRunner', + dataflow_service_account: str = '', + dataflow_disk_size_gb: int = 50, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_workers_num: int = 1, + dataflow_max_workers_num: int = 5, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +): + """Preprocess inputs for information retrieval task - json files for corpus + + and queries, and csv for query to golden docs mapping. + + Args: + project: Required. The GCP project that runs the pipeline component. + location: Required. The GCP region that runs the pipeline component. + corpus_gcs_source: Required. The path for json file containing corpus + documents. + query_gcs_source: Required. The path for json file containing query + documents. + golden_docs_gcs_source: Required. The path for csv file containing mapping + of each query to the golden docs. + embedding_chunking_function: function used to split a document into + chunks. Supported values are `langchain-RecursiveCharacterTextSplitter` + and `sentence-splitter`. + embedding_chunk_size: The length of each document chunk. If 0, chunking is + not enabled. + embedding_chunk_overlap: The length of the overlap part between adjacent + chunks. Will only be used if embedding_chunk_size > 0. + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted + to `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job + should be peered. For example, projects/12345/global/networks/myVPC. + Format is of the form projects/{project}/global/networks/{network}. + Where {project} is a project number, as in 12345, and {network} is a + network name. Private services access must already be configured for the + network. If left unspecified, the job is not peered with any network. + runner: runner for the beam pipeline. DirectRunner and DataflowRunner are + supported. + dataflow_service_account: Service account to run the dataflow job. If not + set, dataflow will use the default worker service account. For more + details, see + https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the + evaluation run. + dataflow_machine_type: The dataflow worker machine type executing the + evaluation run. + dataflow_workers_num: The number of workers executing the evaluation run. + dataflow_max_workers_num: The max number of workers executing the + evaluation run. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when + empty the default subnetwork will be used. More details: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources (str): + Serialized gcp_resources proto tracking the custom job. + predictions_query_gcs_source (list): + The GCS directory to save preprocessed query data to run batch + prediction. + predictions_corpus_gcs_source (list): + The GCS directory to save preprocessed corpus data to run batch + prediction. + embedding_retrieval_gcs_source (str): + The GCS directory to save preprocessed golden docs mapping data to run + batch prediction. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + f'--information_retrieval_preprocessor={True}', + f'--project={project}', + f'--location={location}', + f'--corpus_gcs_source={corpus_gcs_source}', + f'--query_gcs_source={query_gcs_source}', + f'--golden_docs_gcs_source={golden_docs_gcs_source}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--gcp_resources={gcp_resources}', + f'--predictions_query_gcs_source={predictions_query_gcs_source}', + f'--predictions_corpus_gcs_source={predictions_corpus_gcs_source}', + f'--embedding_retrieval_gcs_source={embedding_retrieval_gcs_source}', + f'--embedding_chunking_function={embedding_chunking_function}', + f'--embedding_chunk_size={embedding_chunk_size}', + f'--embedding_chunk_overlap={embedding_chunk_overlap}', + f'--runner={runner}', + f'--dataflow_service_account={dataflow_service_account}', + f'--dataflow_disk_size={dataflow_disk_size_gb}', + f'--dataflow_machine_type={dataflow_machine_type}', + f'--dataflow_workers_num={dataflow_workers_num}', + f'--dataflow_max_workers_num={dataflow_max_workers_num}', + f'--dataflow_subnetwork={dataflow_subnetwork}', + f'--dataflow_use_public_ips={dataflow_use_public_ips}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_retrieval_metrics/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_retrieval_metrics/__init__.py new file mode 100644 index 0000000000..f6ca06c113 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_retrieval_metrics/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Embedding Evaluation Metrics Computation Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_retrieval_metrics/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_retrieval_metrics/component.py new file mode 100644 index 0000000000..e6ffed536d --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_retrieval_metrics/component.py @@ -0,0 +1,127 @@ +"""Information Retrieval metrics computation component used in KFP pipelines.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp.dsl import container_component +from kfp.dsl import Metrics +from kfp.dsl import Output +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +@container_component +def llm_retrieval_metrics( + gcp_resources: OutputPath(str), + retrieval_metrics: Output[Metrics], + project: str, + location: str, + golden_docs_pattern: str, + embedding_retrieval_results_pattern: str, + retrieval_metrics_top_k_list: str, + display_name: str = 'llm_retrieval_metrics_component', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + runner: str = 'DirectRunner', + dataflow_service_account: str = '', + dataflow_disk_size_gb: int = 50, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_workers_num: int = 1, + dataflow_max_workers_num: int = 5, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +): + """Compute retrieval metrics based on the docs retrieved for each query. + + Args: + project: Required. The GCP project that runs the pipeline component. + location: Required. The GCP region that runs the pipeline component. + golden_docs_pattern: Required. Files where queries and corresponding + golden doc ids are saved. The path pattern can contain glob characters + (`*`, `?`, and `[...]` sets). + embedding_retrieval_results_pattern: Required. Files where doc retrieval + results for each query are saved. The path pattern can contain glob + characters (`*`, `?`, and `[...]` sets). + retrieval_metrics_top_k_list: Required. k values for retrieval metrics, + for example, precision@k, accuracy@k, etc. If more than one value, + separated by comma. e.g., "1,5,10". + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted + to `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job + should be peered. For example, projects/12345/global/networks/myVPC. + Format is of the form projects/{project}/global/networks/{network}. + Where {project} is a project number, as in 12345, and {network} is a + network name. Private services access must already be configured for the + network. If left unspecified, the job is not peered with any network. + runner: runner for the beam pipeline. DirectRunner and DataflowRunner are + supported. + dataflow_service_account: Service account to run the dataflow job. If not + set, dataflow will use the default worker service account. For more + details, see + https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the + evaluation run. + dataflow_machine_type: The dataflow worker machine type executing the + evaluation run. + dataflow_workers_num: The number of workers executing the evaluation run. + dataflow_max_workers_num: The max number of workers executing the + evaluation run. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when + empty the default subnetwork will be used. More details: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources: + Serialized gcp_resources proto tracking the custom job. + retrieval_metrics: + A Metrics artifact representing the retrieval metrics. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + f'--retrieval_metrics={True}', + f'--project={project}', + f'--location={location}', + f'--golden_docs_pattern={golden_docs_pattern}', + f'--embedding_retrieval_results_pattern={embedding_retrieval_results_pattern}', + f'--retrieval_metrics_top_k_list={retrieval_metrics_top_k_list}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--gcp_resources={gcp_resources}', + f'--retrieval_metrics_output_path={retrieval_metrics.path}', + f'--runner={runner}', + f'--dataflow_service_account={dataflow_service_account}', + f'--dataflow_disk_size={dataflow_disk_size_gb}', + f'--dataflow_machine_type={dataflow_machine_type}', + f'--dataflow_workers_num={dataflow_workers_num}', + f'--dataflow_max_workers_num={dataflow_max_workers_num}', + f'--dataflow_subnetwork={dataflow_subnetwork}', + f'--dataflow_use_public_ips={dataflow_use_public_ips}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/__init__.py new file mode 100644 index 0000000000..febcd8f962 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline LLM Safety Bias Evaluation Components.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/component.py new file mode 100644 index 0000000000..24ab20fa97 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/component.py @@ -0,0 +1,103 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LLM Safety Bias Metrics component used in KFP pipelines.""" + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from kfp.dsl import Artifact +from kfp.dsl import container_component +from kfp.dsl import Output +from kfp.dsl import OutputPath + +_IMAGE_URI = 'us-docker.pkg.dev/vertex-ai-restricted/llm-eval/llm-bias:v0.2' + + +@container_component +def llm_safety_bias_metrics( + gcp_resources: OutputPath(str), + llm_safety_bias_evaluation_metrics: Output[Artifact], + location: str = 'us-central1', + slice_spec_gcs_source: str = '', + predictions_gcs_source: str = '', + display_name: str = 'llm-safety-bias-component', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +): + """Aggregates LLM safety bias metrics based on specified data slices. + + Args: + location: The GCP region that runs the pipeline components. + slice_spec_gcs_source: Google Cloud Storage location to file with JSONL + slicing spec definition. + predictions_gcs_source: A storage URI pointing toward a GCS file or + directory with prediction results to be used for this evaluation. + display_name: The display name of the evaluation custom job. + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should + be peered. For example, `projects/12345/global/networks/myVPC`. Format is + of the form `projects/{project}/global/networks/{network}`. Where + `{project}` is a project number, as in `12345`, and `{network}` is a + network name, as in `myVPC`. To specify this field, you must have already + configured VPC Network Peering for Vertex AI + (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left + unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options. If set, + resources created by this pipeline will be encrypted with the provided + encryption key. Has the form: + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + The key needs to be in the same region as where the compute resource is + created. + project: The GCP project that runs the pipeline components. Defaults to the + project in which the PipelineJob is run. + + Returns: + llm_safety_bias_evaluation_metrics: `Artifact` tracking the LLM safety + bias evaluation metrics output. + gcp_resources: Serialized gcp_resources proto tracking the custom job. + For more details, see + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=_IMAGE_URI, + args=[ + f'--safety_metrics={True}', + f'--predictions_gcs_source={predictions_gcs_source}', + f'--slice_spec_gcs_source={slice_spec_gcs_source}', + f'--bias_llm_metrics={llm_safety_bias_evaluation_metrics.path}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/evaluation_llm_safety_bias_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/evaluation_llm_safety_bias_pipeline.py new file mode 100644 index 0000000000..c0c5fcf75b --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/llm_safety_bias/evaluation_llm_safety_bias_pipeline.py @@ -0,0 +1,178 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Vertex LLM Safety Bias Evaluation Pipeline.""" + +from typing import NamedTuple + +from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation.llm_safety_bias.component import llm_safety_bias_metrics as LLMSafetyBiasMetricsOp +from google_cloud_pipeline_components.types.artifact_types import VertexBatchPredictionJob +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import ConcatPlaceholder +from kfp.dsl import container_component +from kfp.dsl import ContainerSpec +from kfp.dsl import Output +from kfp.dsl import OutputPath + +_PRIVATE_BP_IMAGE = ( + 'us-docker.pkg.dev/vertex-ai-restricted/llm-eval/private-bp:v0.1' +) + + +@container_component +def private_model_batch_predict( + location: str, + model_name: str, + gcp_resources: OutputPath(str), + batchpredictionjob: Output[VertexBatchPredictionJob], + gcs_output_directory: Output[Artifact], + bp_output_gcs_uri: OutputPath(str), + predictions_format: str = 'jsonl', + job_display_name: str = 'evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + accelerator_type: str = '', + accelerator_count: int = 0, + encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +): + return ContainerSpec( + image=_image.GCPC_IMAGE_TAG, + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--payload', + ConcatPlaceholder([ + '{', + '"display_name": "', + job_display_name, + '", ', + '"job_spec": {"worker_pool_specs": [{"replica_count":"1', + '", "machine_spec": {"machine_type": "e2-standard-4', + '"},', + '"container_spec": {"image_uri":"', + _PRIVATE_BP_IMAGE, + '", "args": ["--project=', + project, + '", "--location=', + location, + '", "--model=', + model_name, + '", "--instances_format=', + 'jsonl', + '", "--predictions_format=', + predictions_format, + '", "--accelerator_type=', + accelerator_type, + '", "--accelerator_count=', + accelerator_count, + '", "--bp_output_gcs_uri=', + bp_output_gcs_uri, + '", "--executor_input={{$.json_escape[1]}}"]}}]', + ', "encryption_spec": {"kms_key_name":"', + encryption_spec_key_name, + '"}', + '}}', + ]), + ], + ) + + +@dsl.pipeline(name='evaluation-llm-safety-bias-pipeline') +def evaluation_llm_safety_bias_pipeline( + project: str, + location: str, + model_name: str, + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +) -> NamedTuple('outputs', llm_safety_bias_evaluation_metrics=Artifact): + """LLM RAI Safety Bias Evaluation pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + model_name: The Model name used to run evaluation. Must be a publisher Model + or a managed Model sharing the same ancestor location. Starting this job + has no impact on any existing deployments of the Model and their + resources. + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should + be peered. For example, `projects/12345/global/networks/myVPC`. Format is + of the form `projects/{project}/global/networks/{network}`. Where + `{project}` is a project number, as in `12345`, and `{network}` is a + network name, as in `myVPC`. To specify this field, you must have already + configured VPC Network Peering for Vertex AI + (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left + unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options. If set, + resources created by this pipeline will be encrypted with the provided + encryption key. Has the form: + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + The key needs to be in the same region as where the compute resource is + created. + + Returns: + llm_safety_bias_evaluation_metrics: Metrics Artifact for LLM Safety Bias. + """ + outputs = NamedTuple( + 'outputs', + llm_safety_bias_evaluation_metrics=Artifact, + ) + batch_predict_task = private_model_batch_predict( + project=project, + location=location, + model_name=model_name, + job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + predictions_format='jsonl', + encryption_spec_key_name=encryption_spec_key_name, + ) + + llm_safety_bias_task = LLMSafetyBiasMetricsOp( + project=project, + predictions_gcs_source=batch_predict_task.outputs['bp_output_gcs_uri'], + slice_spec_gcs_source=( + 'gs://vertex-evaluation-llm-dataset-us-central1/safety_slicing_spec/all_subgroup.json' + ), + location=location, + machine_type=machine_type, + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ) + return outputs( + llm_safety_bias_evaluation_metrics=llm_safety_bias_task.outputs[ + 'llm_safety_bias_evaluation_metrics' + ] + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/target_field_data_remover/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/target_field_data_remover/component.py index cbfcfef14d..5b61856076 100644 --- a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/target_field_data_remover/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/target_field_data_remover/component.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG +from typing import List + +from google_cloud_pipeline_components._implementation.model_evaluation import version from kfp.dsl import container_component from kfp.dsl import ContainerSpec from kfp.dsl import OutputPath @@ -28,7 +30,7 @@ def target_field_data_remover( gcs_output_directory: OutputPath(list), project: str, location: str = 'us-central1', - gcs_source_uris: list = [], + gcs_source_uris: List[str] = [], bigquery_source_uri: str = '', instances_format: str = 'jsonl', target_field_name: str = 'ground_truth', @@ -89,7 +91,7 @@ def target_field_data_remover( """ # fmt: on return ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql/__init__.py new file mode 100644 index 0000000000..d0127e1ee4 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation Text2SQL Pipeline.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql/evaluation_llm_text2sql_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql/evaluation_llm_text2sql_pipeline.py new file mode 100644 index 0000000000..f1f591d681 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql/evaluation_llm_text2sql_pipeline.py @@ -0,0 +1,189 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Text2SQL evaluation pipeline.""" +from typing import Dict + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation.text2sql_evaluation.component import text2sql_evaluation as Text2SQLEvaluationOp +from google_cloud_pipeline_components._implementation.model_evaluation.text2sql_preprocess.component import text2sql_evaluation_preprocess as Text2SQLEvaluationPreprocessOp +from google_cloud_pipeline_components._implementation.model_evaluation.text2sql_validate_and_process.component import text2sql_evaluation_validate_and_process as Text2SQLEvaluationValidateAndProcessOp +from google_cloud_pipeline_components.types import artifact_types +from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp +import kfp +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +_PIPELINE_NAME = 'evaluation_llm_text2sql_pipeline' + + +@kfp.dsl.pipeline(name=_PIPELINE_NAME) +def evaluation_llm_text2sql_pipeline( + model_name: str, + evaluation_data_source_path: str, + tables_metadata_path: str, + prompt_template_path: str = '', + sql_dialect: str = 'bigquery', + evaluation_method: str = 'parser', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, + location: str = _placeholders.LOCATION_PLACEHOLDER, + model_parameters: Dict[str, str] = {}, + batch_predict_instances_format: str = 'jsonl', + batch_predict_predictions_format: str = 'jsonl', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +): + """The LLM Evaluation Text2SQL Pipeline. + + Args: + model_name: The Model used to run text2sql evaluation. Must be a publisher + model or a managed Model sharing the same ancestor location. Starting this + job has no impact on any existing deployments of the Model and their + resources. Supported model is publishers/google/models/text-bison. + evaluation_data_source_path: Required. The path for json file containing + text2sql evaluation input dataset, including natural language question, + ground truth SQL / SQL results. + tables_metadata_path: Required. The path for json file containing database + metadata, including table names, schema fields. + prompt_template_path: Required. The path for json file containing prompt + template. Will provide default value if users do not sepecify. + sql_dialect: Optional. SQL dialect type, e.g. bigquery, mysql, etc. Default + value is bigquery. + evaluation_method: Optional. Text2SQL evaluation method, value can be + 'parser', 'execution', 'all'. Default value is 'parser'. + project: Optional. The GCP project that runs the pipeline components. + Default value is the same project used to run the pipeline. + location: Optional. The GCP region that runs the pipeline components. + Default value is the same location used to run the pipeline. + model_parameters: Optional. The parameters that govern the predictions, e.g. + temperature, + batch_predict_instances_format: The format in which instances are given, + must be one of the Model's supportedInputStorageFormats. If not set, + default to "jsonl". For more details about this input config, see + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + machine_type: The machine type of this custom job. If not set, defaulted to + `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Service account to run the dataflow job. If not set, + dataflow will use the default worker service account. For more details, + see + https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + network: Dataflow's fully qualified subnetwork name, when empty the default + subnetwork will be used. More details: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + """ + + get_vertex_model_task = kfp.dsl.importer( + artifact_uri=( + f'https://{location}-aiplatform.googleapis.com/v1/{model_name}' + ), + artifact_class=artifact_types.VertexModel, + metadata={'resourceName': model_name}, + ) + get_vertex_model_task.set_display_name('get-vertex-model') + + preprocess_task = Text2SQLEvaluationPreprocessOp( + project=project, + location=location, + evaluation_data_source_path=evaluation_data_source_path, + tables_metadata_path=tables_metadata_path, + prompt_template_path=prompt_template_path, + machine_type=machine_type, + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ) + + batch_predict_table_names_task = ModelBatchPredictOp( + job_display_name='text2sql-batch-prediction-table-names-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + model=get_vertex_model_task.outputs['artifact'], + location=location, + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_source_uris=preprocess_task.outputs['model_inference_input_path'], + model_parameters=model_parameters, + gcs_destination_output_uri_prefix=( + f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_table_names_output' + ), + encryption_spec_key_name=encryption_spec_key_name, + project=project, + ) + + validate_table_names_and_process_task = Text2SQLEvaluationValidateAndProcessOp( + project=project, + location=location, + model_inference_type='table_name_case', + model_inference_results_directory=batch_predict_table_names_task.outputs[ + 'gcs_output_directory' + ], + tables_metadata_path=tables_metadata_path, + prompt_template_path=prompt_template_path, + machine_type=machine_type, + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ) + + batch_predict_column_names_task = ModelBatchPredictOp( + job_display_name='text2sql-batch-prediction-column-names-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + model=get_vertex_model_task.outputs['artifact'], + location=location, + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_source_uris=validate_table_names_and_process_task.outputs[ + 'model_inference_input_path' + ], + model_parameters=model_parameters, + gcs_destination_output_uri_prefix=( + f'{PIPELINE_ROOT_PLACEHOLDER}/batch_predict_column_names_output' + ), + encryption_spec_key_name=encryption_spec_key_name, + project=project, + ) + + _ = Text2SQLEvaluationValidateAndProcessOp( + project=project, + location=location, + model_inference_type='column_name_case', + model_inference_results_directory=batch_predict_column_names_task.outputs[ + 'gcs_output_directory' + ], + tables_metadata_path=tables_metadata_path, + prompt_template_path=prompt_template_path, + machine_type=machine_type, + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ) + + _ = Text2SQLEvaluationOp( + project=project, + location=location, + sql_dialect=sql_dialect, + evaluation_method=evaluation_method, + # TODO(bozhengbz) Change value to model_inference_results_directory + # when sql query model batch prediction component is added. + model_inference_results_directory=batch_predict_table_names_task.outputs[ + 'gcs_output_directory' + ], + tables_metadata_path=tables_metadata_path, + machine_type=machine_type, + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_evaluation/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_evaluation/__init__.py new file mode 100644 index 0000000000..5e23a86fc7 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_evaluation/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Text2SQL Evaluation Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_evaluation/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_evaluation/component.py new file mode 100644 index 0000000000..a084de02d4 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_evaluation/component.py @@ -0,0 +1,118 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Text2SQL evaluation component used in KFP pipelines.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp.dsl import Artifact +from kfp.dsl import container_component +from kfp.dsl import Input +from kfp.dsl import Metrics +from kfp.dsl import Output +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +@container_component +def text2sql_evaluation( + gcp_resources: OutputPath(str), + error_analysis_path: OutputPath(str), + generated_sql_results_path: OutputPath(str), + text2sql_evaluation_metrics: Output[Metrics], + project: str, + location: str, + sql_dialect: str, + evaluation_method: str, + model_inference_results_directory: Input[Artifact], + tables_metadata_path: str, + display_name: str = 'text2sql-evaluation', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +): + """Get evaluation metrics, generated SQL results and error analysis in text2sql evaluation pipeline. + + Args: + project: Required. The GCP project that runs the pipeline component. + location: Required. The GCP region that runs the pipeline component. + sql_dialect: Required. SQL dialect type, e.g. bigquery, mysql, etc. + evaluation_method: Required. Text2SQL evaluation method, value can be + 'parser', 'execution', 'all'. + model_inference_results_directory: Required. The path for json file + containing text2sql model inference results from the last step. + tables_metadata_path: Required. The path for json file containing database + metadata, including table names, schema fields. + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted + to `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job + should be peered. For example, projects/12345/global/networks/myVPC. + Format is of the form projects/{project}/global/networks/{network}. + Where {project} is a project number, as in 12345, and {network} is a + network name. Private services access must already be configured for the + network. If left unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources (str): + Serialized gcp_resources proto tracking the custom job. + error_analysis_path: + Path for aggregrated error analysis for genereated SQL queris. + generated_sql_results_path: + Path for generated sql queries and execution results (optional) + in json format. + text2sql_evaluation_metrics: + A Metrics artifact representing the text2sql evaluation metrics. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + f'--text2sql_evaluation={True}', + f'--project={project}', + f'--location={location}', + f'--sql_dialect={sql_dialect}', + f'--evaluation_method={evaluation_method}', + f'--model_inference_results_directory={model_inference_results_directory.path}', + f'--tables_metadata_path={tables_metadata_path}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--gcp_resources={gcp_resources}', + f'--error_analysis_path={error_analysis_path}', + f'--generated_sql_results_path={generated_sql_results_path}', + f'--text2sql_evaluation_metrics_output_path={text2sql_evaluation_metrics.path}', + f'--error_analysis_path={error_analysis_path}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_preprocess/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_preprocess/__init__.py new file mode 100644 index 0000000000..198fd983f0 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_preprocess/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Text2SQL Evaluation Preprocess Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_preprocess/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_preprocess/component.py new file mode 100644 index 0000000000..583da4c23b --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_preprocess/component.py @@ -0,0 +1,103 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Text2SQL evaluation preprocess component used in KFP pipelines.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp.dsl import container_component +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +@container_component +def text2sql_evaluation_preprocess( + gcp_resources: OutputPath(str), + model_inference_input_path: OutputPath(list), + project: str, + location: str, + evaluation_data_source_path: str, + tables_metadata_path: str, + prompt_template_path: str = '', + display_name: str = 'text2sql-evaluation-preprocess', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +): + """Preprocess inputs for text2sql evaluation pipeline. + + Args: + project: Required. The GCP project that runs the pipeline component. + location: Required. The GCP region that runs the pipeline component. + evaluation_data_source_path: Required. The path for json file containing + text2sql evaluation input dataset, including natural language question, + ground truth SQL / SQL results. + tables_metadata_path: Required. The path for json file containing database + metadata, including table names, schema fields. + prompt_template_path: Required. The path for json file containing prompt + template. Will provide default value if users do not sepecify. + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted + to `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job + should be peered. For example, projects/12345/global/networks/myVPC. + Format is of the form projects/{project}/global/networks/{network}. + Where {project} is a project number, as in 12345, and {network} is a + network name. Private services access must already be configured for the + network. If left unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources (str): + Serialized gcp_resources proto tracking the custom job. + model_inference_input_path (list): + The GCS path to save preprocessed data to run batch + prediction to get table names. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + f'--text2sql_preprocess={True}', + f'--project={project}', + f'--location={location}', + f'--evaluation_data_source_path={evaluation_data_source_path}', + f'--tables_metadata_path={tables_metadata_path}', + f'--prompt_template_path={prompt_template_path}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--gcp_resources={gcp_resources}', + f'--model_inference_input_path={model_inference_input_path}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_validate_and_process/__init__.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_validate_and_process/__init__.py new file mode 100644 index 0000000000..7980146e7d --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_validate_and_process/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Evaluation LLM Text2SQL Evaluation Validate and Process Component.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_validate_and_process/component.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_validate_and_process/component.py new file mode 100644 index 0000000000..3f1b097262 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/text2sql_validate_and_process/component.py @@ -0,0 +1,109 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Text2SQL evaluation preprocess component used in KFP pipelines.""" + +from google_cloud_pipeline_components import utils as gcpc_utils +from google_cloud_pipeline_components._implementation.model_evaluation import utils +from google_cloud_pipeline_components._implementation.model_evaluation import version +from kfp.dsl import Artifact +from kfp.dsl import container_component +from kfp.dsl import Input +from kfp.dsl import OutputPath +from kfp.dsl import PIPELINE_ROOT_PLACEHOLDER + + +@container_component +def text2sql_evaluation_validate_and_process( + gcp_resources: OutputPath(str), + model_inference_input_path: OutputPath(list), + project: str, + location: str, + model_inference_type: str, + model_inference_results_directory: Input[Artifact], + tables_metadata_path: str, + prompt_template_path: str = '', + display_name: str = 'text2sql-evaluation-validate-and-process', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', +): + """Text2SQL evaluation component to validate model inference results in previous step and generate model inference input in the next step. + + Args: + project: Required. The GCP project that runs the pipeline component. + location: Required. The GCP region that runs the pipeline component. + model_inference_type: Required. Model inference type to differentiate + model inference results validataion steps, values can be table_name_case + or column_name_case. + model_inference_results_directory: Required. The directory to store all of + files containing text2sql model inference results from the last step. + tables_metadata_path: Required. The path for json file containing database + metadata, including table names, schema fields. + prompt_template_path: Required. The path for json file containing prompt + template. Will provide default value if users do not sepecify. + display_name: The name of the Evaluation job. + machine_type: The machine type of this custom job. If not set, defaulted + to `e2-highmem-16`. More details: + https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job + should be peered. For example, projects/12345/global/networks/myVPC. + Format is of the form projects/{project}/global/networks/{network}. + Where {project} is a project number, as in 12345, and {network} is a + network name. Private services access must already be configured for the + network. If left unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + gcp_resources (str): + Serialized gcp_resources proto tracking the custom job. + model_inference_input_path (str): + The GCS path to save processed data to run batch prediction in the + next step. + """ + return gcpc_utils.build_serverless_customjob_container_spec( + project=project, + location=location, + custom_job_payload=utils.build_custom_job_payload( + display_name=display_name, + machine_type=machine_type, + image_uri=version.LLM_EVAL_IMAGE_TAG, + args=[ + f'--text2sql_validate_and_process={True}', + f'--project={project}', + f'--location={location}', + f'--model_inference_type={model_inference_type}', + f'--model_inference_results_directory={model_inference_results_directory.path}', + f'--tables_metadata_path={tables_metadata_path}', + f'--prompt_template_path={prompt_template_path}', + f'--root_dir={PIPELINE_ROOT_PLACEHOLDER}', + f'--gcp_resources={gcp_resources}', + f'--model_inference_input_path={model_inference_input_path}', + '--executor_input={{$.json_escape[1]}}', + ], + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ), + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/utils.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/utils.py new file mode 100644 index 0000000000..3be30d9952 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/utils.py @@ -0,0 +1,80 @@ +"""Utility functions used to create custom Kubeflow components.""" + +from typing import Any, Dict, List + +from google_cloud_pipeline_components import _image + + +def build_custom_job_payload( + *, + display_name: str, + image_uri: str, + args: List[str], + machine_type: str = 'n1-standard-4', + service_account: str = '', + network: str = '', + reserved_ip_ranges: List[str] = [], + accelerator_type: str = 'ACCELERATOR_TYPE_UNSPECIFIED', + accelerator_count: int = 0, + encryption_spec_key_name: str = '', +) -> Dict[str, Any]: + """Generates payload for a CustomJob in a Sec4 horizontal compliant way. + + Args: + display_name: CustomJob display name. Can contain up to 128 UTF-8 + characters. + image_uri: Docker image URI to use for the CustomJob. + args: Arguments to pass to the Docker image. + machine_type: The type of the machine. See the list of machine types + supported for custom training: + https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types + service_account: Sets the default service account for workload run-as + account. The service account running the pipeline + (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) + submitting jobs must have act-as permission on this run-as account. If + unspecified, the Vertex AI Custom Code Service + Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) + for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should + be peered. For example, projects/12345/global/networks/myVPC. Format is of + the form projects/{project}/global/networks/{network}. Where {project} is + a project number, as in 12345, and {network} is a network name. Private + services access must already be configured for the network. If left + unspecified, the job is not peered with any network. + reserved_ip_ranges: A list of names for the reserved ip ranges under the VPC + network that can be used for this job. If set, we will deploy the job + within the provided ip ranges. Otherwise, the job will be deployed to any + ip ranges under the provided VPC network. + accelerator_type: The type of accelerator(s) that may be attached to the + machine as per acceleratorCount. + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec#AcceleratorType + accelerator_count: The number of accelerators to attach to the machine. + encryption_spec_key_name: Customer-managed encryption key options for the + CustomJob. If this is set, then all resources created by the CustomJob + will be encrypted with the provided encryption key. + + Returns: + CustomJob payload dictionary. + """ + payload = { + 'display_name': str(display_name), + 'job_spec': { + 'worker_pool_specs': [{ + 'replica_count': '1', + 'machine_spec': { + 'machine_type': str(machine_type), + 'accelerator_type': str(accelerator_type), + 'accelerator_count': int(accelerator_count), + }, + 'container_spec': { + 'image_uri': image_uri, + 'args': args, + }, + }], + 'service_account': str(service_account), + 'network': str(network), + 'reserved_ip_ranges': reserved_ip_ranges, + }, + 'encryption_spec': {'kms_key_name': str(encryption_spec_key_name)}, + } + return payload diff --git a/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/version.py b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/version.py new file mode 100644 index 0000000000..f5bc1ab908 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_implementation/model_evaluation/version.py @@ -0,0 +1,24 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Version constants for model evaluation components.""" + +_EVAL_VERSION = 'v0.9.3' +_LLM_EVAL_VERSION = 'v0.3' + +_EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/model-evaluation' +_LLM_EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/llm-model-evaluation' + + +EVAL_IMAGE_TAG = f'{_EVAL_IMAGE_NAME}:{_EVAL_VERSION}' +LLM_EVAL_IMAGE_TAG = f'{_LLM_EVAL_IMAGE_NAME}:{_LLM_EVAL_VERSION}' diff --git a/components/google-cloud/google_cloud_pipeline_components/_placeholders.py b/components/google-cloud/google_cloud_pipeline_components/_placeholders.py new file mode 100644 index 0000000000..44dd3cae8a --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/_placeholders.py @@ -0,0 +1,18 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Placeholders for use in component authoring.""" + +# prefer not using placeholder suffix like KFP does for reduce verbosity +PROJECT_ID_PLACEHOLDER = "{{$.pipeline_google_cloud_project_id}}" +LOCATION_PLACEHOLDER = "{{$.pipeline_google_cloud_location}}" diff --git a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/__init__.py b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/__init__.py new file mode 100644 index 0000000000..7ef02c25d0 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline LLM Components container code.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/__init__.py b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/__init__.py new file mode 100644 index 0000000000..f3293916fb --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Components - LLM Templated Custom Job Remote Launcher. + +Note: The `v1/custom_job/` launcher is the preferred Custom Job launcher +whenever possible, because this Jinja2-based templated launcher does not take +advantage of the Vertex Pipeline backend optimization and will thus launch a +Custom Job that runs a Custom Job. +""" diff --git a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/launcher.py b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/launcher.py new file mode 100644 index 0000000000..6fa0a0deb9 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/launcher.py @@ -0,0 +1,139 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Jinja2-based templated launcher for custom jobs. + +Note: Whenever possible, please prefer the vanilla custom_job launcher instead +of this Jinja2-based templated custom job launcher. This launcher does not take +advantage of the Vertex Pipeline backend optimization and will thus launch a +Custom Job that runs another Custom Job. +""" + +import argparse +import logging +import sys +from typing import Any, Dict, List + +from google_cloud_pipeline_components.container._implementation.llm.templated_custom_job import remote_runner +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import parser_util + + +def _parse_args(args: List[str]) -> Dict[str, Any]: + """Parse command line arguments.""" + parser, _ = parser_util.parse_default_args(args) + parser.add_argument( + '--dry_run', + dest='dry_run', + action='store_true', + help=( + 'If set, log the rendered payload for the Custom Job and exit with ' + 'error code 1' + ), + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--set_integer', + dest='set_integer', + action='append', + nargs='+', + help='(key, [value]) pairs. If `value` is missing, the value is None', + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--set_string', + dest='set_string', + action='append', + nargs='+', + help='(key, [value]) pairs. If `value` is missing, the value is None', + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--set_float', + dest='set_float', + action='append', + nargs='+', + help='(key, [value]) pairs. If `value` is missing, the value is None', + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--set_boolean', + dest='set_boolean', + action='append', + nargs='+', + help='(key, [value]) pairs. If `value` is missing, the value is None', + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--set_json', + dest='set_json', + action='append', + nargs='+', + help='(key, [value]) pairs. If `value` is missing, the value is None', + default=argparse.SUPPRESS, + ) + parsed_args, _ = parser.parse_known_args(args) + return vars(parsed_args) + + +def main(argv: List[str]) -> None: + """Main entry. + + Expected input args are as follows: + project - Required. The project of which the resource will be launched. + location - Required. The region of which the resource will be launched. + type - Required. GCPC launcher is a single container. This Enum will specify + which resource to be launched. + payload - Required. The base64-encoded Jinja2 payload that will be rendered + to a full serialized JSON of the resource spec. This payload normally + doesn't contain Pipeline Placehoders. + gcp_resources - Required. Placeholder output for returning job_id. + dry_run - If set, log the rendered payload for the Custom Job and exit with + error code 1. + set_integer - A list of `(key, [value])` pairs of strings that'll be used to + render the payload. The `value` will be cast to an integer. If `value` is + missing, it'll be `None`. Note that `value` can contain Pipeline + Placeholders. + set_string - A list of (key, [value]) pairs of strings that'll be used to + render the payload. If `value` is missing, it'll be `None`. Note that + `value` can contain Pipeline Placeholders. + set_float - A list of `(key, [value])` pairs of strings that'll be used to + render the payload. The `value` will be cast to a float number. If `value` + is missing, it'll be `None`. Note that `value` can contain Pipeline + Placeholders. + set_boolean - A list of `(key, [value])` pairs of strings that'll be used to + render the payload. The `value` will be cast to a boolean value. If + `value` is missing, it'll be `None`. Note that `value` can contain + Pipeline Placeholders. + set_json - A list of `(key, [value])` pairs of strings that'll be used to + render the payload. The `value` will be cast to an object by calling + `json.loads()`. If `value` is missing, it'll be `None`. Note that `value` + can contain Pipeline Placeholders. + + Args: + argv: A list of system arguments. + """ + parsed_args = _parse_args(argv) + logging.basicConfig( + format='[%(asctime)s] [%(levelname)s]: %(message)s', level=logging.INFO + ) + job_type = parsed_args['type'] + if job_type != 'TemplatedCustomJob': + raise ValueError('Incorrect job type: ' + job_type) + + logging.info('Job started for type: %s', job_type) + + remote_runner.create_templated_custom_job(**parsed_args) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/remote_runner.py new file mode 100644 index 0000000000..3423181124 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/llm/templated_custom_job/remote_runner.py @@ -0,0 +1,207 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GCP remote runner for templated custom jobs based on the AI Platform SDK.""" + +import base64 +import json +import logging +import sys +from typing import Any, Callable, Dict, List, Optional +import google_cloud_pipeline_components.google_cloud_pipeline_components.container.v1.custom_job.remote_runner as custom_job_remote_runner +import jinja2 +from jinja2 import sandbox + +# Note that type annotations need to match the python version in the GCPC docker +# image in addition to the internal python version. + +ParamListType = Optional[List[List[str]]] + + +def _json_escape_filter(value: str) -> str: + """A Jinja2 filter for JSON escaping.""" + return json.dumps(value)[1:-1] + + +def render_payload(payload_str: str, params: Dict[str, Any]) -> str: + """Renders a base64-encoded Custom Job payload in Jinja2 format.""" + env = sandbox.ImmutableSandboxedEnvironment( + autoescape=False, undefined=jinja2.StrictUndefined + ) + + # We add an additional `json_dumps` filter because the builtin filter + # `tojson`, which is implemented as `htmlsafe_json_dumps`, converts special + # symbols to `\u` format, e.g., `&` -> `\u0026`, `<` -> `\u003c`. + env.filters['json_dumps'] = json.dumps + env.filters['json_escape'] = _json_escape_filter + + template = env.from_string(payload_str) + return template.render(**params) + + +def convert_key_value_param_list( + param_list: Optional[List[List[str]]], + type_cast: Callable[[str], Any], + cmd_flag: str, +) -> Dict[str, Any]: + """Converts a list of (key, [value]) pairs to a dictionary. + + Args: + param_list: A list of (key, [value]) pairs of string type. + type_cast: A function to cast `value`, if exists, from string to a + specific type. + cmd_flag: The command-line flag for this list of parameters. This is used + to provide better error message when raising an exception. + + Returns: + A dictionary of the converted (key, value) pairs. + """ + params = {} + if not param_list: + return params + for key_value in param_list: + if 1 <= len(key_value) <= 2: + key = key_value[0] + if 1 == len(key_value): + params[key] = None + else: + try: + params[key] = type_cast(key_value[1]) + except json.JSONDecodeError as e: + raise ValueError( + f'Cannot decode value for [{key}]: {key_value[1]}' + ) from e + else: + raise ValueError( + f'{cmd_flag} can only take 1 or 2 params, but found {key_value}' + ) + return params + + +def convert_integer_params( + integer_params: ParamListType, +) -> Dict[str, Optional[int]]: + """Converts a list of (key, [integer]) pairs to a dictionary.""" + return convert_key_value_param_list( + param_list=integer_params, type_cast=int, cmd_flag='--set_integer' + ) + + +def convert_string_params( + string_params: ParamListType, +) -> Dict[str, Optional[str]]: + """Converts a list of (key, [string]) pairs to a dictionary.""" + return convert_key_value_param_list( + param_list=string_params, type_cast=str, cmd_flag='--set_string' + ) + + +def convert_float_params( + float_params: ParamListType, +) -> Dict[str, Optional[float]]: + """Converts a list of (key, [float]) pairs to a dictionary.""" + return convert_key_value_param_list( + param_list=float_params, type_cast=float, cmd_flag='--set_float' + ) + + +def convert_boolean_params( + boolean_params: ParamListType, +) -> Dict[str, Optional[bool]]: + """Converts a list of (key, [boolean]) pairs to a dictionary.""" + return convert_key_value_param_list( + param_list=boolean_params, + type_cast=lambda x: x.lower() in ['1', 'true', 'y', 'yes'], + cmd_flag='--set_boolean', + ) + + +def convert_json_params( + json_params: ParamListType, +) -> Dict[str, Any]: + """Converts a list of (key, [json objects]) pairs to a dictionary.""" + return convert_key_value_param_list( + param_list=json_params, type_cast=json.loads, cmd_flag='--set_json' + ) + + +# This method will also be used for unit tests. +def decode_and_render_payload( + payload: str, + int_params: ParamListType = None, + string_params: ParamListType = None, + float_params: ParamListType = None, + boolean_params: ParamListType = None, + json_params: ParamListType = None, +) -> str: + """Decodes base64-encoded Jinja2 payload and renders it.""" + params = convert_integer_params(int_params) + params.update(convert_string_params(string_params)) + params.update(convert_float_params(float_params)) + params.update(convert_boolean_params(boolean_params)) + params.update(convert_json_params(json_params)) + + return render_payload(base64.b64decode(payload).decode('utf-8'), params) + + +def create_templated_custom_job( + type: str, # pylint: disable=redefined-builtin + project: str, + location: str, + payload: str, + gcp_resources: str, + dry_run: bool = False, + set_integer: ParamListType = None, + set_string: ParamListType = None, + set_float: ParamListType = None, + set_boolean: ParamListType = None, + set_json: ParamListType = None, +) -> None: + """Creates and polls a Custom Job.""" + rendered_payload = decode_and_render_payload( + payload=payload, + int_params=set_integer, + string_params=set_string, + float_params=set_float, + boolean_params=set_boolean, + json_params=set_json, + ) + + # Call json.loads() to validate that the payload is a valid JSON. + # Call json.dumps() instead of using rendered_payload to remove redundant + # blank spaces in the payload. + try: + payload_str = json.dumps(json.loads(rendered_payload)) + except json.JSONDecodeError as e: + logging.error( + 'Cannot deserialize the rendered payload to JSON: %r', rendered_payload + ) + raise ValueError( + 'The rendered payload is an invalid JSON. Please see the error log for ' + 'details.' + ) from e + + if dry_run: + logging.info( + 'Log rendered payload for dry run and exit with error code 1: %s', + payload_str, + ) + sys.exit(1) + + custom_job_remote_runner.create_custom_job( + type=type, + project=project, + location=location, + payload=payload_str, + gcp_resources=gcp_resources, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model/get_model/get_model.py b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model/get_model/get_model.py index e3aaf662f1..b8cc1ced75 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model/get_model/get_model.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model/get_model/get_model.py @@ -20,7 +20,7 @@ from google.api_core import gapic_v1 from google.cloud import aiplatform -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.proto.gcp_resources_pb2 import GcpResources from google_cloud_pipeline_components.types.artifact_types import VertexModel @@ -120,7 +120,7 @@ def _get_model( 'model', vertex_uri_prefix + model_resource_name, model_resource_name ) # TODO(b/266848949): Output Artifact should use correct MLMD artifact. - artifact_util.update_output_artifacts(executor_input, [vertex_model]) + artifact_utils.update_output_artifacts(executor_input, [vertex_model]) resources = GcpResources() model_resource = resources.resources.add() diff --git a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model_evaluation/import_model_evaluation.py b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model_evaluation/import_model_evaluation.py index 1bcf928eb6..d82b8340e2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model_evaluation/import_model_evaluation.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/_implementation/model_evaluation/import_model_evaluation.py @@ -36,12 +36,15 @@ 'text-generation': 'gs://google-cloud-aiplatform/schema/modelevaluation/general_text_generation_metrics_1.0.0.yaml', 'question-answering': 'gs://google-cloud-aiplatform/schema/modelevaluation/question_answering_metrics_1.0.0.yaml', 'summarization': 'gs://google-cloud-aiplatform/schema/modelevaluation/summarization_metrics_1.0.0.yaml', + 'embedding': 'gs://google-cloud-aiplatform/schema/modelevaluation/embedding_metrics_1.0.0.yaml', } MODEL_EVALUATION_RESOURCE_TYPE = 'ModelEvaluation' MODEL_EVALUATION_SLICE_RESOURCE_TYPE = 'ModelEvaluationSlice' SLICE_BATCH_IMPORT_LIMIT = 50 -ULM_TASKS = set(['text-generation', 'question-answering', 'summarization']) +ULM_TASKS = set( + ['text-generation', 'question-answering', 'summarization', 'embedding'] +) def _make_parent_dirs_and_return_path(file_path: str): @@ -83,6 +86,12 @@ def _make_parent_dirs_and_return_path(file_path: str): type=str, default='', ) +parser.add_argument( + '--embedding_metrics', + dest='embedding_metrics', + type=str, + default='', +) parser.add_argument( '--feature_attributions', dest='feature_attributions', @@ -129,13 +138,20 @@ def _make_parent_dirs_and_return_path(file_path: str): required=True, default=argparse.SUPPRESS, ) +parser.add_argument( + '--evaluation_resource_name', + dest='evaluation_resource_name', + type=_make_parent_dirs_and_return_path, + required=True, + default=argparse.SUPPRESS, +) def main(argv): """Calls ModelService.ImportModelEvaluation.""" parsed_args, _ = parser.parse_known_args(argv) - if parsed_args.model_name.startswith('publishers'): + if 'publishers/google' in parsed_args.model_name: return _, project_id, _, location, _, model_id = parsed_args.model_name.split('/') @@ -160,9 +176,17 @@ def main(argv): elif parsed_args.summarization_metrics: metrics_file_path = parsed_args.summarization_metrics problem_type = 'summarization' + elif parsed_args.embedding_metrics: + metrics_file_path = parsed_args.embedding_metrics + problem_type = 'embedding' else: metrics_file_path = parsed_args.metrics problem_type = parsed_args.problem_type + if problem_type not in PROBLEM_TYPE_TO_SCHEMA_URI: + raise ValueError( + 'Unsupported problem_type: {}. Supported problem types are: {}' + .format(problem_type, list(PROBLEM_TYPE_TO_SCHEMA_URI.keys())) + ) logging.info('metrics_file_path: %s', metrics_file_path) logging.info('problem_type: %s', problem_type) @@ -270,6 +294,10 @@ def main(argv): ) model_evaluation_name = import_model_evaluation_response.name + # Write the model evaluation resource to evaluation_resource_name output. + with open(parsed_args.evaluation_resource_name, 'w') as f: + f.write(model_evaluation_name) + resources = GcpResources() # Write the model evaluation resource to GcpResources output. model_eval_resource = resources.resources.add() diff --git a/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/__init__.py b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/__init__.py new file mode 100644 index 0000000000..11b29089f1 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Components - Custom Job Launcher and Remote Runner.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/launcher.py b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/launcher.py new file mode 100644 index 0000000000..d9c7237421 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/launcher.py @@ -0,0 +1,56 @@ +# Copyright 2021 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GCP launcher for custom jobs based on the AI Platform SDK.""" + +import logging +import sys + +from google_cloud_pipeline_components.container.preview.custom_job import remote_runner +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import parser_util + + +def _parse_args(args): + """Parse command line arguments.""" + _, parsed_args = parser_util.parse_default_args(args) + return vars(parsed_args) + + +def main(argv): + """Main entry. + + Expected input args are as follows: + Project - Required. The project of which the resource will be launched. + Region - Required. The region of which the resource will be launched. + Type - Required. GCP launcher is a single container. This Enum will + specify which resource to be launched. + Request payload - Required. The full serialized json of the resource spec. + Note this can contain the Pipeline Placeholders. + gcp_resources - placeholder output for returning job_id. + + Args: + argv: A list of system arguments. + """ + parsed_args = _parse_args(argv) + job_type = parsed_args['type'] + + if job_type != 'CustomJob': + raise ValueError('Incorrect job type: ' + job_type) + + logging.info('Job started for type: ' + job_type) + + remote_runner.create_custom_job(**parsed_args) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py new file mode 100644 index 0000000000..d72f35833a --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/preview/custom_job/remote_runner.py @@ -0,0 +1,97 @@ +# Copyright 2022 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GCP launcher for custom jobs based on the AI Platform SDK.""" + +import json + +from google.api_core import retry +from google_cloud_pipeline_components.container.preview.gcp_launcher import job_remote_runner +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import gcp_labels_util + +_CUSTOM_JOB_RETRY_DEADLINE_SECONDS = 10.0 * 60.0 +LABELS_PAYLOAD_KEY = 'labels' + + +def insert_system_labels_into_payload(payload): + job_spec = json.loads(payload) + job_spec[LABELS_PAYLOAD_KEY] = gcp_labels_util.attach_system_labels( + job_spec[LABELS_PAYLOAD_KEY] if LABELS_PAYLOAD_KEY in job_spec else {} + ) + return json.dumps(job_spec) + + +def create_custom_job_with_client(job_client, parent, job_spec): + create_custom_job_fn = None + try: + create_custom_job_fn = job_client.create_custom_job( + parent=parent, custom_job=job_spec + ) + except (ConnectionError, RuntimeError) as err: + error_util.exit_with_internal_error(err.args[0]) + return create_custom_job_fn + + +def get_custom_job_with_client(job_client, job_name): + get_custom_job_fn = None + try: + get_custom_job_fn = job_client.get_custom_job( + name=job_name, + retry=retry.Retry(deadline=_CUSTOM_JOB_RETRY_DEADLINE_SECONDS), + ) + except (ConnectionError, RuntimeError) as err: + error_util.exit_with_internal_error(err.args[0]) + return get_custom_job_fn + + +def create_custom_job( + type, + project, + location, + payload, + gcp_resources, +): + """Create and poll custom job status till it reaches a final state. + + This follows the typical launching logic: + 1. Read if the custom job already exists in gcp_resources + - If already exists, jump to step 3 and poll the job status. This happens + if the launcher container experienced unexpected termination, such as + preemption + 2. Deserialize the payload into the job spec and create the custom job + 3. Poll the custom job status every _POLLING_INTERVAL_IN_SECONDS seconds + - If the custom job is succeeded, return succeeded + - If the custom job is cancelled/paused, it's an unexpected scenario so + return failed + - If the custom job is running, continue polling the status + Also retry on ConnectionError up to + job_remote_runner._CONNECTION_ERROR_RETRY_LIMIT times during the poll. + """ + remote_runner = job_remote_runner.JobRemoteRunner( + type, project, location, gcp_resources + ) + + try: + # Create custom job if it does not exist + job_name = remote_runner.check_if_job_exists() + if job_name is None: + job_name = remote_runner.create_job( + create_custom_job_with_client, + insert_system_labels_into_payload(payload), + ) + + # Poll custom job status until "JobState.JOB_STATE_SUCCEEDED" + remote_runner.poll_job(get_custom_job_with_client, job_name) + except (ConnectionError, RuntimeError) as err: + error_util.exit_with_internal_error(err.args[0]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/preview/gcp_launcher/__init__.py b/components/google-cloud/google_cloud_pipeline_components/container/preview/gcp_launcher/__init__.py new file mode 100644 index 0000000000..4baaf5d93f --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/preview/gcp_launcher/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GCP Launcher Modules for google cloud components.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/container/preview/gcp_launcher/job_remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/preview/gcp_launcher/job_remote_runner.py new file mode 100644 index 0000000000..1ff4e6fd8e --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/preview/gcp_launcher/job_remote_runner.py @@ -0,0 +1,235 @@ +# Copyright 2021 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common module for launching and managing the Vertex Job resources.""" + +import json +import logging +import os +from os import path +import re +import time +from typing import Optional + +from google.api_core import client_options +from google.api_core import gapic_v1 +import google.auth +import google.auth.transport.requests +from google.cloud import aiplatform_v1beta1 +from google.cloud.aiplatform_v1beta1.types import job_state as gca_job_state +from google_cloud_pipeline_components.container.utils import execution_context +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util +from google_cloud_pipeline_components.proto.gcp_resources_pb2 import GcpResources +import requests + +from google.protobuf import json_format + +_POLLING_INTERVAL_IN_SECONDS = 20 +_CONNECTION_ERROR_RETRY_LIMIT = 5 + +_JOB_COMPLETE_STATES = ( + gca_job_state.JobState.JOB_STATE_SUCCEEDED, + gca_job_state.JobState.JOB_STATE_FAILED, + gca_job_state.JobState.JOB_STATE_CANCELLED, + gca_job_state.JobState.JOB_STATE_PAUSED, +) + +_JOB_ERROR_STATES = ( + gca_job_state.JobState.JOB_STATE_FAILED, + gca_job_state.JobState.JOB_STATE_CANCELLED, + gca_job_state.JobState.JOB_STATE_PAUSED, +) + +# Job error codes mapping can be found in: +# https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto +_JOB_USER_ERROR_CODES = ( + 3, # INVALID_ARGUMENT + 5, # NOT_FOUND + 7, # PERMISSION_DENIED + 6, # ALREADY_EXISTS + 9, # FAILED_PRECONDITION + 11, # OUT_OF_RANGE + 12, # UNIMPLEMENTED +) + + +class JobRemoteRunner: + """Common module for creating and poll jobs on the Vertex Platform.""" + + def __init__(self, job_type, project, location, gcp_resources): + """Initializes a job client and other common attributes.""" + self.job_type = job_type + self.project = project + self.location = location + self.gcp_resources = gcp_resources + self.client_options = client_options.ClientOptions( + api_endpoint=location + '-aiplatform.googleapis.com' + ) + self.client_info = gapic_v1.client_info.ClientInfo( + user_agent='google-cloud-pipeline-components' + ) + self.job_client = aiplatform_v1beta1.JobServiceClient( + client_options=self.client_options, client_info=self.client_info + ) + self.job_uri_prefix = f'https://{self.client_options.api_endpoint}/v1beta1/' + self.poll_job_name = '' + + def check_if_job_exists(self) -> Optional[str]: + """Check if the job already exists.""" + if ( + path.exists(self.gcp_resources) + and os.stat(self.gcp_resources).st_size != 0 + ): + with open(self.gcp_resources) as f: + serialized_gcp_resources = f.read() + job_resources = json_format.Parse( + serialized_gcp_resources, GcpResources() + ) + # Resources should only contain one item. + if len(job_resources.resources) != 1: + raise ValueError( + 'gcp_resources should contain one resource, found' + f' {len(job_resources.resources)}' + ) + + job_name_group = re.findall( + f'{self.job_uri_prefix}(.*)', + job_resources.resources[0].resource_uri, + ) + + if not job_name_group or not job_name_group[0]: + raise ValueError( + 'Job Name in gcp_resource is not formatted correctly or is empty.' + ) + job_name = job_name_group[0] + + logging.info( + '%s name already exists: %s. Continue polling the status', + self.job_type, + job_name, + ) + return job_name + else: + return None + + def create_job(self, create_job_fn, payload) -> str: + """Create a job.""" + parent = f'projects/{self.project}/locations/{self.location}' + # TODO(kevinbnaughton) remove empty fields from the spec temporarily. + job_spec = json_util.recursive_remove_empty( + json.loads(payload, strict=False) + ) + create_job_response = create_job_fn(self.job_client, parent, job_spec) + job_name = create_job_response.name + + # Write the job proto to output. + job_resources = GcpResources() + job_resource = job_resources.resources.add() + job_resource.resource_type = self.job_type + job_resource.resource_uri = f'{self.job_uri_prefix}{job_name}' + + with open(self.gcp_resources, 'w') as f: + f.write(json_format.MessageToJson(job_resources)) + + return job_name + + def poll_job(self, get_job_fn, job_name: str): + """Poll the job status.""" + with execution_context.ExecutionContext( + on_cancel=lambda: self.send_cancel_request(job_name) + ): + retry_count = 0 + while True: + try: + get_job_response = get_job_fn(self.job_client, job_name) + retry_count = 0 + # Handle transient connection error. + except ConnectionError as err: + retry_count += 1 + if retry_count < _CONNECTION_ERROR_RETRY_LIMIT: + logging.warning( + ( + 'ConnectionError (%s) encountered when polling job: %s.' + ' Trying to recreate the API client.' + ), + err, + job_name, + ) + # Recreate the Python API client. + self.job_client = aiplatform_v1beta1.JobServiceClient( + self.client_options, self.client_info + ) + logging.info( + 'Waiting for %s seconds for next poll.', + _POLLING_INTERVAL_IN_SECONDS, + ) + time.sleep(_POLLING_INTERVAL_IN_SECONDS) + continue + else: + # TODO(ruifang) propagate the error. + # Exit with an internal error code. + error_util.exit_with_internal_error( + f'Request failed after {_CONNECTION_ERROR_RETRY_LIMIT} retries.' + ) + + if get_job_response.state == gca_job_state.JobState.JOB_STATE_SUCCEEDED: + logging.info( + 'Get%s response state =%s', self.job_type, get_job_response.state + ) + return get_job_response + elif get_job_response.state in _JOB_ERROR_STATES: + # TODO(ruifang) propagate the error. + if get_job_response.error.code in _JOB_USER_ERROR_CODES: + raise ValueError( + 'Job failed with value error in error state: {}.'.format( + get_job_response.state + ) + ) + else: + raise RuntimeError( + 'Job failed with error state: {}.'.format( + get_job_response.state + ) + ) + else: + logging.info( + ( + 'Job %s is in a non-final state %s.' + ' Waiting for %s seconds for next poll.' + ), + job_name, + get_job_response.state, + _POLLING_INTERVAL_IN_SECONDS, + ) + time.sleep(_POLLING_INTERVAL_IN_SECONDS) + + def send_cancel_request(self, job_name: str): + if not job_name: + return + creds, _ = google.auth.default( + scopes=['https://www.googleapis.com/auth/cloud-platform'] + ) + if not creds.valid: + creds.refresh(google.auth.transport.requests.Request()) + headers = { + 'Content-type': 'application/json', + 'Authorization': 'Bearer ' + creds.token, + } + # Vertex AI cancel APIs: + # https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.hyperparameterTuningJobs/cancel + # https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.customJobs/cancel + # https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs/cancel + requests.post( + url=f'{self.job_uri_prefix}{job_name}:cancel', data='', headers=headers + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/utils/artifact_util.py b/components/google-cloud/google_cloud_pipeline_components/container/utils/artifact_utils.py similarity index 95% rename from components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/utils/artifact_util.py rename to components/google-cloud/google_cloud_pipeline_components/container/utils/artifact_utils.py index 59fccb2458..45dab98864 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/utils/artifact_util.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/utils/artifact_utils.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Utilities for working with artifacts.""" + import json import os @@ -21,7 +23,7 @@ def update_output_artifact( target_artifact_name: str, uri: str, metadata: dict = {}, -): +) -> None: """Updates the output artifact with the new uri and metadata.""" executor_input_json = json.loads(executor_input) executor_output = {'artifacts': {}} @@ -47,7 +49,10 @@ def update_output_artifact( # Writes a list of Artifacts to the executor output file. -def update_output_artifacts(executor_input: str, artifacts: list): +def update_output_artifacts( + executor_input: str, + artifacts: list, +) -> None: """Updates a list of Artifacts to the executor output file.""" executor_input_json = json.loads(executor_input) executor_output = {'artifacts': {}} diff --git a/components/google-cloud/google_cloud_pipeline_components/container/utils/execution_context.py b/components/google-cloud/google_cloud_pipeline_components/container/utils/execution_context.py index c336cb8033..f43b53680f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/utils/execution_context.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/utils/execution_context.py @@ -21,7 +21,7 @@ class ExecutionContext: """Execution context for running inside Google Cloud Pipeline Components. The base class is aware of the GCPC environment and can cascade - a pipeline cancel event to the operation through ``on_cancel`` handler. + a pipeline cancel event to the operation through `on_cancel` handler. Args: on_cancel: optional, function to handle KFP cancel event. """ diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/__init__.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/__init__.py new file mode 100644 index 0000000000..cbe4dd022e --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Google Cloud Pipeline Components - AutoML Image Training Job Launcher and Remote Runner.""" diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/launcher.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/launcher.py new file mode 100644 index 0000000000..1662994efa --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/launcher.py @@ -0,0 +1,236 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GCP launcher for AutoML image training jobs based on the AI Platform SDK.""" + +import argparse +import json +import logging +import sys +from typing import List + +from google_cloud_pipeline_components.container.v1.automl_training_job.image import remote_runner +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import parser_util + + +def _parse_args(args: List[str]): + """Parse command line arguments.""" + args.append('--payload') + args.append('"{}"') # Unused but required by parser_util. + parser, _ = parser_util.parse_default_args(args) + # Parse the conditionally required arguments + parser.add_argument( + '--display_name', + dest='display_name', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--prediction_type', + dest='prediction_type', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--multi_label', + dest='multi_label', + type=parser_util.parse_bool, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--model_type', + dest='model_type', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--labels', + dest='labels', + type=json.loads, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--dataset', + dest='dataset', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--disable_early_stopping', + dest='disable_early_stopping', + type=parser_util.parse_bool, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--training_encryption_spec_key_name', + dest='training_encryption_spec_key_name', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--model_encryption_spec_key_name', + dest='model_encryption_spec_key_name', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--model_display_name', + dest='model_display_name', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--training_fraction_split', + dest='training_fraction_split', + type=float, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--validation_fraction_split', + dest='validation_fraction_split', + type=float, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--test_fraction_split', + dest='test_fraction_split', + type=float, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--budget_milli_node_hours', + dest='budget_milli_node_hours', + type=int, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--training_filter_split', + dest='training_filter_split', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--validation_filter_split', + dest='validation_filter_split', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--test_filter_split', + dest='test_filter_split', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--base_model', + dest='base_model', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--incremental_train_base_model', + dest='incremental_train_base_model', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--parent_model', + dest='parent_model', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--is_default_version', + dest='is_default_version', + type=parser_util.parse_bool, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--model_version_aliases', + dest='model_version_aliases', + type=json.loads, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--model_version_description', + dest='model_version_description', + type=str, + required=False, + default=argparse.SUPPRESS, + ) + parser.add_argument( + '--model_labels', + dest='model_labels', + type=json.loads, + required=False, + default=argparse.SUPPRESS, + ) + parsed_args, _ = parser.parse_known_args(args) + args_dict = vars(parsed_args) + del args_dict['payload'] + return args_dict + + +def main(argv: List[str]): + """Main entry. + + Expected input args are as follows: + Project - Required. The project of which the resource will be launched. + Region - Required. The region of which the resource will be launched. + Type - Required. GCP launcher is a single container. This Enum will + specify which resource to be launched. + gcp_resources - placeholder output for returning job_id. + Extra arguments - For constructing request payload. See remote_runner.py for + more information. + + Args: + argv: A list of system arguments. + """ + parsed_args = _parse_args(argv) + job_type = parsed_args['type'] + + if job_type != 'AutoMLImageTrainingJob': + raise ValueError('Incorrect job type: ' + job_type) + + logging.info( + 'Starting AutoMLImageTrainingJob using the following arguments: %s', + parsed_args, + ) + + remote_runner.create_pipeline(**parsed_args) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/remote_runner.py new file mode 100644 index 0000000000..a48616cc23 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/automl_training_job/image/remote_runner.py @@ -0,0 +1,250 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GCP remote runner for AutoML image training pipelines based on the AI Platform SDK.""" + +import logging +from typing import Any, Dict, Optional, Sequence + +from google.api_core import retry +from google.cloud.aiplatform import datasets +from google.cloud.aiplatform import gapic +from google.cloud.aiplatform import initializer +from google.cloud.aiplatform import models +from google.cloud.aiplatform import schema +from google.cloud.aiplatform import training_jobs +from google.cloud.aiplatform_v1.types import model +from google.cloud.aiplatform_v1.types import training_pipeline +from google_cloud_pipeline_components.container.v1.gcp_launcher import pipeline_remote_runner +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util + +from google.protobuf import struct_pb2 +from google.protobuf import json_format + + +_GET_PIPELINE_RETRY_DEADLINE_SECONDS = 10.0 * 60.0 +_CLASSIFICATION = 'classification' +_OBJECT_DETECTION = 'object_detection' + + +# pylint: disable=protected-access +def create_payload( + project: str, + location: str, + display_name: Optional[str] = None, + prediction_type: str = _CLASSIFICATION, + multi_label: bool = False, + model_type: str = 'CLOUD', + labels: Optional[Dict[str, str]] = None, + dataset: Optional[str] = None, + disable_early_stopping: bool = False, + training_encryption_spec_key_name: Optional[str] = None, + model_encryption_spec_key_name: Optional[str] = None, + model_display_name: Optional[str] = None, + training_fraction_split: Optional[float] = None, + validation_fraction_split: Optional[float] = None, + test_fraction_split: Optional[float] = None, + budget_milli_node_hours: Optional[int] = None, + training_filter_split: Optional[str] = None, + validation_filter_split: Optional[str] = None, + test_filter_split: Optional[str] = None, + base_model: Optional[str] = None, + incremental_train_base_model: Optional[str] = None, + parent_model: Optional[str] = None, + is_default_version: Optional[bool] = True, + model_version_aliases: Optional[Sequence[str]] = None, + model_version_description: Optional[str] = None, + model_labels: Optional[Dict[str, str]] = None, +) -> str: + """Creates a AutoML Image Training Job payload.""" + # Override default model_type for object_detection + if model_type == 'CLOUD' and prediction_type == _OBJECT_DETECTION: + model_type = 'CLOUD_HIGH_ACCURACY_1' + + training_encryption_spec = initializer.global_config.get_encryption_spec( + encryption_spec_key_name=training_encryption_spec_key_name + ) + model_encryption_spec = initializer.global_config.get_encryption_spec( + encryption_spec_key_name=model_encryption_spec_key_name + ) + + # Training task inputs. + training_task_inputs = { + # required inputs + 'modelType': model_type, + 'budgetMilliNodeHours': budget_milli_node_hours, + # optional inputs + 'disableEarlyStopping': disable_early_stopping, + } + if prediction_type == _CLASSIFICATION: + training_task_inputs['multiLabel'] = multi_label + if incremental_train_base_model: + training_task_inputs['uptrainBaseModelId'] = incremental_train_base_model + + training_task_definition = getattr( + schema.training_job.definition, f'automl_image_{prediction_type}' + ) + + # Input data config. + input_data_config = training_jobs._TrainingJob._create_input_data_config( + dataset=dataset and datasets.ImageDataset(dataset_name=dataset), + training_fraction_split=training_fraction_split, + validation_fraction_split=validation_fraction_split, + test_fraction_split=test_fraction_split, + training_filter_split=training_filter_split, + validation_filter_split=validation_filter_split, + test_filter_split=test_filter_split, + ) + + # Model to upload. + model_to_upload = model.Model( + display_name=model_display_name or display_name, + labels=model_labels or labels, + encryption_spec=model_encryption_spec, + version_aliases=models.ModelRegistry._get_true_alias_list( + model_version_aliases, is_default_version + ), + version_description=model_version_description, + ) + + # Sets base_model. + if base_model: + training_task_inputs['baseModelId'] = base_model + + # Create training task inputs. + training_task_inputs_struct = struct_pb2.Struct() + training_task_inputs_struct.update(training_task_inputs) + + # Gets parent_model. + parent_model = models.ModelRegistry._get_true_version_parent( + parent_model=parent_model, + project=project, + location=location, + ) + + pipeline = training_pipeline.TrainingPipeline( + display_name=display_name, + training_task_definition=training_task_definition, + training_task_inputs=struct_pb2.Value( + struct_value=training_task_inputs_struct + ), + model_to_upload=model_to_upload, + parent_model=parent_model, + input_data_config=input_data_config, + labels=labels, + encryption_spec=training_encryption_spec, + ) + + return json_format.MessageToJson( + pipeline._pb, preserving_proto_field_name=True + ) + + +# pylint: enable=protected-access + + +def create_pipeline_with_client( + pipeline_client: gapic.PipelineServiceClient, + parent, + pipeline_spec: Any, +): + """Creates a training pipeline with the client.""" + created_pipeline = None + try: + logging.info( + 'Creating AutoML Vision training pipeline with sanitized pipeline' + ' spec: %s', + pipeline_spec, + ) + created_pipeline = pipeline_client.create_training_pipeline( + parent=parent, + training_pipeline=training_pipeline.TrainingPipeline(**pipeline_spec), + ) + except (ConnectionError, RuntimeError) as err: + error_util.exit_with_internal_error(err.args[0]) + return created_pipeline + + +def get_pipeline_with_client( + pipeline_client: gapic.PipelineServiceClient, pipeline_name: str +): + """Gets training pipeline state with the client.""" + get_automl_vision_training_pipeline = None + try: + get_automl_vision_training_pipeline = pipeline_client.get_training_pipeline( + name=pipeline_name, + retry=retry.Retry(deadline=_GET_PIPELINE_RETRY_DEADLINE_SECONDS), + ) + except (ConnectionError, RuntimeError) as err: + error_util.exit_with_internal_error(err.args[0]) + return get_automl_vision_training_pipeline + + +def create_pipeline( + type: str, # pylint: disable=redefined-builtin + project: str, + location: str, + gcp_resources: str, + **kwargs: Dict[str, Any], +): + """Create and poll AutoML Vision training pipeline status till it reaches a final state. + + This follows the typical launching logic: + 1. Read if the training pipeline already exists in gcp_resources + - If already exists, jump to step 3 and poll the pipeline status. This + happens + if the launcher container experienced unexpected termination, such as + preemption + 2. Deserialize the payload into the pipeline spec and create the training + pipeline + 3. Poll the training pipeline status every + pipeline_remote_runner._POLLING_INTERVAL_IN_SECONDS seconds + - If the training pipeline is succeeded, return succeeded + - If the training pipeline is cancelled/paused, it's an unexpected + scenario so return failed + - If the training pipeline is running, continue polling the status + + Also retry on ConnectionError up to + pipeline_remote_runner._CONNECTION_ERROR_RETRY_LIMIT times during the poll. + + Args: + type: Job type. + project: Project name. + location: Location to start the training job. + gcp_resources: URI for storing GCP resources. + **kwargs: Extra args for creating the payload. + """ + remote_runner = pipeline_remote_runner.PipelineRemoteRunner( + type, project, location, gcp_resources + ) + + try: + # Create AutoML vision training pipeline if it does not exist + pipeline_name = remote_runner.check_if_pipeline_exists() + if pipeline_name is None: + payload = create_payload(project, location, **kwargs) + logging.info( + 'AutoML Vision training payload formatted: %s', + payload, + ) + pipeline_name = remote_runner.create_pipeline( + create_pipeline_with_client, + payload, + ) + + # Poll AutoML Vision training pipeline status until + # "PipelineState.PIPELINE_STATE_SUCCEEDED" + remote_runner.poll_pipeline(get_pipeline_with_client, pipeline_name) + + except (ConnectionError, RuntimeError) as err: + error_util.exit_with_internal_error(err.args[0]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/batch_prediction_job/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/batch_prediction_job/remote_runner.py index cb872700ad..8a0a0f9468 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/batch_prediction_job/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/batch_prediction_job/remote_runner.py @@ -19,8 +19,8 @@ from google.api_core import retry from google.cloud.aiplatform import explain +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.gcp_launcher import job_remote_runner -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import gcp_labels_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util @@ -241,6 +241,6 @@ def create_batch_prediction_job( ) ) - artifact_util.update_output_artifacts(executor_input, output_artifacts) + artifact_utils.update_output_artifacts(executor_input, output_artifacts) except (ConnectionError, RuntimeError) as err: error_util.exit_with_internal_error(err.args[0]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/create_model/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/create_model/remote_runner.py index 42122d78b0..9fa0e21a48 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/create_model/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/create_model/remote_runner.py @@ -15,8 +15,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util from google_cloud_pipeline_components.types.artifact_types import BQMLModel @@ -96,4 +96,4 @@ def bigquery_create_model_job( # tableId is the model ID modelId = query_result['ddlTargetTable']['tableId'] bqml_model_artifact = BQMLModel.create('model', projectId, datasetId, modelId) - artifact_util.update_output_artifacts(executor_input, [bqml_model_artifact]) + artifact_utils.update_output_artifacts(executor_input, [bqml_model_artifact]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/evaluate_model/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/evaluate_model/remote_runner.py index 7eab4354e2..c80666cd92 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/evaluate_model/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/evaluate_model/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_evaluate_model_job( @@ -126,7 +126,7 @@ def bigquery_evaluate_model_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'evaluation_metrics', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/feature_importance/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/feature_importance/remote_runner.py index 3327f28917..adc1d136e5 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/feature_importance/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/feature_importance/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_feature_importance_job( @@ -83,7 +83,7 @@ def bigquery_ml_feature_importance_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'feature_importance', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_advanced_weights/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_advanced_weights/remote_runner.py index d5e6d41f18..cb43dbf1f9 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_advanced_weights/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_advanced_weights/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_advanced_weights_job( @@ -90,7 +90,7 @@ def bigquery_ml_advanced_weights_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'advanced_weights', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_coefficients/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_coefficients/remote_runner.py index d39c42599d..9d2c877602 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_coefficients/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_coefficients/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_arima_coefficients( @@ -78,7 +78,7 @@ def bigquery_ml_arima_coefficients( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'arima_coefficients', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_evaluate/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_evaluate/remote_runner.py index 43fc52282e..f9ae51d4fe 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_evaluate/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_arima_evaluate/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_arima_evaluate_job( @@ -103,7 +103,7 @@ def bigquery_ml_arima_evaluate_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'arima_evaluation_metrics', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_centroids/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_centroids/remote_runner.py index e7ba48e4ed..ac426c91c0 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_centroids/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_centroids/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_centroids_job( @@ -100,7 +100,7 @@ def bigquery_ml_centroids_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'centroids', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_feature_info/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_feature_info/remote_runner.py index b1e37597e0..cc6a808293 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_feature_info/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_feature_info/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_feature_info_job( @@ -90,7 +90,7 @@ def bigquery_ml_feature_info_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'feature_info', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_training_info/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_training_info/remote_runner.py index 7128287923..a2677989d4 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_training_info/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_training_info/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_training_info_job( @@ -90,7 +90,7 @@ def bigquery_ml_training_info_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'ml_training_info', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_trial_info/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_trial_info/remote_runner.py index 37fbf5edcb..56c9587a45 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_trial_info/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_trial_info/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_trial_info_job( @@ -94,7 +94,7 @@ def bigquery_ml_trial_info_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'trial_info', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_weights/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_weights/remote_runner.py index 7e3b0b1219..85b64b95c1 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_weights/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/ml_weights/remote_runner.py @@ -17,8 +17,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.bigquery.utils import bigquery_util -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util def bigquery_ml_weights_job( @@ -90,7 +90,7 @@ def bigquery_ml_weights_job( query_results = bigquery_util.get_query_results( project, job_id, location, creds ) - artifact_util.update_output_artifact( + artifact_utils.update_output_artifact( executor_input, 'weights', '', diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/utils/bigquery_util.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/utils/bigquery_util.py index fe1bc44b5a..cbfce404c9 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/utils/bigquery_util.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/bigquery/utils/bigquery_util.py @@ -22,8 +22,8 @@ import google.auth import google.auth.transport.requests +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.utils import execution_context -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import gcp_labels_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util from google_cloud_pipeline_components.proto import gcp_resources_pb2 @@ -334,4 +334,4 @@ def bigquery_query_job( bq_table_artifact = BQTable.create( artifact_name, projectId, datasetId, tableId ) - artifact_util.update_output_artifacts(executor_input, [bq_table_artifact]) + artifact_utils.update_output_artifacts(executor_input, [bq_table_artifact]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/create_endpoint/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/create_endpoint/remote_runner.py index 36edb1711e..f92ebbd4e8 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/create_endpoint/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/create_endpoint/remote_runner.py @@ -13,8 +13,8 @@ # limitations under the License. import json +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import gcp_labels_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util from google_cloud_pipeline_components.types.artifact_types import VertexEndpoint @@ -56,4 +56,4 @@ def create_endpoint( vertex_uri_prefix + endpoint_resource_name, endpoint_resource_name, ) - artifact_util.update_output_artifacts(executor_input, [vertex_endpoint]) + artifact_utils.update_output_artifacts(executor_input, [vertex_endpoint]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/undeploy_model/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/undeploy_model/remote_runner.py index 65e79e3958..01218c1245 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/undeploy_model/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/endpoint/undeploy_model/remote_runner.py @@ -19,7 +19,6 @@ from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util - _MODEL_NAME_TEMPLATE = r'(projects/(?P.*)/locations/(?P.*)/models/(?P.*))' _ENDPOINT_NAME_TEMPLATE = r'(projects/(?P.*)/locations/(?P.*)/endpoints/(?P.*))' @@ -60,19 +59,35 @@ def undeploy_model( get_endpoint_remote_runner = lro_remote_runner.LroRemoteRunner(location) endpoint = get_endpoint_remote_runner.request(get_endpoint_url, '', 'get') + # may or may not contain a model version + full_model_name = undeploy_model_request['model'] + + if '@' in full_model_name: + model_name_no_version, model_version = full_model_name.rsplit('@') + else: + model_name_no_version = full_model_name + model_version = None + # Get the deployed_model_id deployed_model_id = '' - model_name = undeploy_model_request['model'] for deployed_model in endpoint['deployedModels']: - if deployed_model['model'] == model_name: + if deployed_model['model'] == model_name_no_version and ( + # undeploy if version is unspecified + model_version is None + # or version matches + or deployed_model['modelVersionId'] == model_version + ): deployed_model_id = deployed_model['id'] break + print('deployed_model_id', deployed_model_id) if not deployed_model_id: # TODO(ruifang) propagate the error. raise ValueError( - 'Model {} not found at endpoint {}.'.format(model_name, endpoint_name) + 'Model {} not found at endpoint {}.'.format( + full_model_name, endpoint_name + ) ) # Undeploy the model @@ -85,14 +100,14 @@ def undeploy_model( ] model_uri_pattern = re.compile(_MODEL_NAME_TEMPLATE) - match = model_uri_pattern.match(model_name) + match = model_uri_pattern.match(model_name_no_version) try: location = match.group('location') except AttributeError as err: # TODO(ruifang) propagate the error. raise ValueError( 'Invalid model name: {}. Expect: {}.'.format( - model_name, + full_model_name, 'projects/[project_id]/locations/[location]/models/[model_id]', ) ) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/pipeline_remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/pipeline_remote_runner.py index 8a807d590a..c5c77c2975 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/pipeline_remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/pipeline_remote_runner.py @@ -29,13 +29,15 @@ from google.cloud.aiplatform_v1.types import pipeline_job from google.cloud.aiplatform_v1.types import pipeline_state from google.cloud.aiplatform_v1.types import training_pipeline +from google_cloud_pipeline_components.container.utils import execution_context +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util +from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util from google_cloud_pipeline_components.proto import gcp_resources_pb2 import requests from google.rpc import code_pb2 from google.protobuf import json_format -from google_cloud_pipeline_components.container.utils import execution_context -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util, error_util + _POLLING_INTERVAL_IN_SECONDS = 20 _CONNECTION_ERROR_RETRY_LIMIT = 5 @@ -231,6 +233,7 @@ def poll_pipeline( error_util.exit_with_internal_error( f'Request failed after {_CONNECTION_ERROR_RETRY_LIMIT} retries.' ) + return # Not necessary, only to please the linter. if ( get_pipeline_response.state diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/utils/parser_util.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/utils/parser_util.py index a1f2e0f48a..a2f4bfe434 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/utils/parser_util.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/gcp_launcher/utils/parser_util.py @@ -61,3 +61,7 @@ def parse_default_args(args): parsed_args, _ = parser.parse_known_args(args) return (parser, parsed_args) + + +def parse_bool(value: str) -> bool: + return value is not None and value.lower() != 'false' diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/model/delete_model/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/model/delete_model/remote_runner.py index 4798fbc630..b73b010d90 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/model/delete_model/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/model/delete_model/remote_runner.py @@ -14,6 +14,7 @@ import json import re + from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util @@ -22,12 +23,12 @@ def delete_model( - type, - project, - location, - payload, - gcp_resources, -): + type: str, + project: str, + location: str, + payload: str, + gcp_resources: str, +) -> None: """Delete model and poll the LongRunningOperator till it reaches a final state.""" # TODO(IronPan) temporarily remove the empty fields from the spec delete_model_request = json_util.recursive_remove_empty( @@ -49,13 +50,15 @@ def delete_model( ) api_endpoint = location + '-aiplatform.googleapis.com' vertex_uri_prefix = f'https://{api_endpoint}/v1/' - delete_model_url = f'{vertex_uri_prefix}{model_name}' + # vertex AI delete model API deletes the full model, not only the version + model_name_no_version = model_name.rsplit('@', 1)[0] + delete_model_url = f'{vertex_uri_prefix}{model_name_no_version}' try: remote_runner = lro_remote_runner.LroRemoteRunner(location) delete_model_lro = remote_runner.create_lro( delete_model_url, '', gcp_resources, 'delete' ) - delete_model_lro = remote_runner.poll_lro(lro=delete_model_lro) + remote_runner.poll_lro(lro=delete_model_lro) except (ConnectionError, RuntimeError) as err: error_util.exit_with_internal_error(err.args[0]) diff --git a/components/google-cloud/google_cloud_pipeline_components/container/v1/model/upload_model/remote_runner.py b/components/google-cloud/google_cloud_pipeline_components/container/v1/model/upload_model/remote_runner.py index ebb6c2c8db..3a88048805 100644 --- a/components/google-cloud/google_cloud_pipeline_components/container/v1/model/upload_model/remote_runner.py +++ b/components/google-cloud/google_cloud_pipeline_components/container/v1/model/upload_model/remote_runner.py @@ -13,16 +13,15 @@ # limitations under the License. import json -from typing import Optional +from typing import Any, Dict, Optional +from google_cloud_pipeline_components.container.utils import artifact_utils from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner -from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import artifact_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import error_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import gcp_labels_util from google_cloud_pipeline_components.container.v1.gcp_launcher.utils import json_util from google_cloud_pipeline_components.types.artifact_types import VertexModel - ARTIFACT_PROPERTY_KEY_UNMANAGED_CONTAINER_MODEL = 'unmanaged_container_model' API_KEY_PREDICT_SCHEMATA = 'predict_schemata' API_KEY_CONTAINER_SPEC = 'container_spec' @@ -30,7 +29,9 @@ _LABELS_PAYLOAD_KEY = 'labels' -def append_unmanaged_model_artifact_into_payload(executor_input, model_spec): +def append_unmanaged_model_artifact_into_payload( + executor_input: str, model_spec: Dict[str, Any] +) -> Dict[str, Any]: artifact = ( json.loads(executor_input) .get('inputs', {}) @@ -54,14 +55,14 @@ def append_unmanaged_model_artifact_into_payload(executor_input, model_spec): def upload_model( - type, - project, - location, - payload, - gcp_resources, - executor_input, + type: str, + project: str, + location: str, + payload: str, + gcp_resources: str, + executor_input: str, parent_model_name: Optional[str] = None, -): +) -> None: """Upload model and poll the LongRunningOperator till it reaches a final state.""" api_endpoint = location + '-aiplatform.googleapis.com' vertex_uri_prefix = f'https://{api_endpoint}/v1/' @@ -100,14 +101,14 @@ def upload_model( ) upload_model_lro = remote_runner.poll_lro(lro=upload_model_lro) model_resource_name = upload_model_lro['response']['model'] - if 'model_version_id' in upload_model_lro['response']: + if 'modelVersionId' in upload_model_lro['response']: model_resource_name += ( - f'@{upload_model_lro["response"]["model_version_id"]}' + f'@{upload_model_lro["response"]["modelVersionId"]}' ) vertex_model = VertexModel.create( 'model', vertex_uri_prefix + model_resource_name, model_resource_name ) - artifact_util.update_output_artifacts(executor_input, [vertex_model]) + artifact_utils.update_output_artifacts(executor_input, [vertex_model]) except (ConnectionError, RuntimeError) as err: error_util.exit_with_internal_error(err.args[0]) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/__init__.py new file mode 100644 index 0000000000..aa8704bef8 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/__init__.py new file mode 100644 index 0000000000..917eb0e145 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/__init__.py @@ -0,0 +1,65 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Experimental AutoML forecasting components.""" +import os + +from google_cloud_pipeline_components.preview.automl.forecasting.forecasting_ensemble import automl_forecasting_ensemble as ForecastingEnsembleOp +from google_cloud_pipeline_components.preview.automl.forecasting.forecasting_stage_1_tuner import automl_forecasting_stage_1_tuner as ForecastingStage1TunerOp +from google_cloud_pipeline_components.preview.automl.forecasting.forecasting_stage_2_tuner import automl_forecasting_stage_2_tuner as ForecastingStage2TunerOp +from kfp import components + +__all__ = [ + 'ForecastingStage1TunerOp', + 'ForecastingEnsembleOp', + 'ForecastingStage2TunerOp', + 'learn_to_learn_forecasting_pipeline', + 'sequence_to_sequence_forecasting_pipeline', + 'temporal_fusion_transformer_forecasting_pipeline', + 'time_series_dense_encoder_forecasting_pipeline', +] + +learn_to_learn_forecasting_pipeline = components.load_component_from_file( + # Note, please don't name it as `component.yaml` which will conflict with + # the generated file. + os.path.join( + os.path.dirname(__file__), 'learn_to_learn_forecasting_pipeline.yaml' + ) +) + +sequence_to_sequence_forecasting_pipeline = components.load_component_from_file( + # Note, please don't name it as `component.yaml` which will conflict with + # the generated file. + os.path.join( + os.path.dirname(__file__), + 'sequence_to_sequence_forecasting_pipeline.yaml', + ) +) + +temporal_fusion_transformer_forecasting_pipeline = components.load_component_from_file( + # Note, please don't name it as `component.yaml` which will conflict with + # the generated file. + os.path.join( + os.path.dirname(__file__), + 'temporal_fusion_transformer_forecasting_pipeline.yaml', + ) +) + +time_series_dense_encoder_forecasting_pipeline = components.load_component_from_file( + # Note, please don't name it as `component.yaml` which will conflict with + # the generated file. + os.path.join( + os.path.dirname(__file__), + 'time_series_dense_encoder_forecasting_pipeline.yaml', + ) +) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py new file mode 100644 index 0000000000..d683aeceef --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_ensemble.py @@ -0,0 +1,141 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Forecasting Ensemble component spec.""" + +from typing import Optional + +from google_cloud_pipeline_components import utils +from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +# pylint: disable=g-bare-generic,g-doc-args,unused-argument +@dsl.container_component +def automl_forecasting_ensemble( + project: str, + location: str, + root_dir: str, + transform_output: Input[Artifact], + metadata: Input[Artifact], + tuning_result_input: Input[Artifact], + instance_baseline: Input[Artifact], + instance_schema_path: Input[Artifact], + prediction_image_uri: str, + gcp_resources: dsl.OutputPath(str), + model_architecture: Output[Artifact], + example_instance: Output[Artifact], + unmanaged_container_model: Output[UnmanagedContainerModel], + explanation_metadata: dsl.OutputPath(dict), + explanation_metadata_artifact: Output[Artifact], + explanation_parameters: dsl.OutputPath(dict), + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Ensembles AutoML Forecasting models. + + Args: + project: Project to run the job in. + location: Region to run the job in. + root_dir: The Cloud Storage path to store the output. + transform_output: The transform output artifact. + metadata: The tabular example gen metadata. + tuning_result_input: AutoML Tabular tuning result. + instance_baseline: The instance baseline used to calculate explanations. + instance_schema_path: The path to the instance schema, describing the input data for the tf_model at serving time. + encryption_spec_key_name: Customer-managed encryption key. + prediction_image_uri: URI of the Docker image to be used as the container for serving predictions. This URI must identify an image in Artifact Registry or Container Registry. + + Returns: + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + model_architecture: The architecture of the output model. + unmanaged_container_model: Model information needed to perform batch prediction. + explanation_metadata: The explanation metadata used by Vertex online and batch explanations. + explanation_metadata_artifact: The explanation metadata used by Vertex online and batch explanations in the format of a KFP Artifact. + explanation_parameters: The explanation parameters used by Vertex online and batch explanations. + example_instance: An example instance which may be used as an input for predictions. + """ + # fmt: on + job_id = dsl.PIPELINE_JOB_ID_PLACEHOLDER + task_id = dsl.PIPELINE_TASK_ID_PLACEHOLDER + image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125' + display_name = f'automl-forecasting-ensemble-{job_id}-{task_id}' + + error_file_path = f'{root_dir}/{job_id}/{task_id}/error.pb' + model_relative_path = f'{job_id}/{task_id}/model' + explanation_metadata_paths = ( + f'{explanation_metadata},{explanation_metadata_artifact.uri}' + ) + + job_args = [ + 'forecasting_mp_ensemble', + f'--transform_output_path={transform_output.uri}', + f'--error_file_path={error_file_path}', + f'--metadata_path={metadata.uri}', + f'--tuning_result_input_path={tuning_result_input.uri}', + f'--instance_baseline_path={instance_baseline.uri}', + f'--instance_schema_path={instance_schema_path.uri}', + f'--prediction_docker_uri={prediction_image_uri}', + f'--model_relative_output_path={model_relative_path}', + f'--explanation_metadata_path={explanation_metadata_paths}', + f'--explanation_parameters_path={explanation_parameters}', + f'--model_architecture_path={model_architecture.uri}', + f'--example_instance_path={example_instance.uri}', + '--use_json=true', + '--executor_input={{$.json_escape[1]}}', + ] + + payload = { + 'display_name': display_name, + 'encryption_spec': { + 'kms_key_name': encryption_spec_key_name, + }, + 'job_spec': { + 'worker_pool_specs': [{ + 'replica_count': 1, + 'machine_spec': { + 'machine_type': 'n1-highmem-8', + }, + 'container_spec': { + 'image_uri': f'{image_uri}', + 'args': job_args, + }, + }] + }, + } + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + utils.container_component_dumps(payload), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py new file mode 100644 index 0000000000..aaef5f438d --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_1_tuner.py @@ -0,0 +1,148 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Forecasting Stage 1 Tuner component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +# pylint: disable=dangerous-default-value,g-bare-generic,g-doc-args,unused-argument +@dsl.container_component +def automl_forecasting_stage_1_tuner( + project: str, + location: str, + root_dir: str, + num_selected_trials: int, + deadline_hours: float, + num_parallel_trials: int, + single_run_max_secs: int, + metadata: Input[Artifact], + transform_output: Input[Artifact], + materialized_train_split: Input[Artifact], + materialized_eval_split: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + tuning_result_output: Output[Artifact], + study_spec_parameters_override: Optional[list] = [], + worker_pool_specs_override_json: Optional[list] = [], + reduce_search_space_mode: Optional[str] = 'regular', + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Searches AutoML Forecasting architectures and selects the top trials. + + Args: + project: Project to run hyperparameter tuning. + location: Location for running the hyperparameter tuning. + root_dir: The Cloud Storage location to store the output. + study_spec_parameters_override: JSON study spec. E.g., [{"parameter_id": "activation","categorical_value_spec": {"values": ["tanh"]}}] + worker_pool_specs_override_json: JSON worker pool specs. E.g., [{"machine_spec": {"machine_type": "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": "n1-standard-16"}}] + reduce_search_space_mode: The reduce search space mode. Possible values: "regular" (default), "minimal", "full". + num_selected_trials: Number of selected trials. The number of weak learners in the final model is 5 * num_selected_trials. + deadline_hours: Number of hours the hyperparameter tuning should run. + num_parallel_trials: Number of parallel training trials. + single_run_max_secs: Max number of seconds each training trial runs. + metadata: The tabular example gen metadata. + transform_output: The transform output artifact. + materialized_train_split: The materialized train split. + materialized_eval_split: The materialized eval split. + encryption_spec_key_name: Customer-managed encryption key. + + Returns: + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + tuning_result_output: The trained model and architectures. + """ + # fmt: on + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "automl-forecasting-stage-1-tuner-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125', + '", "args": ["forecasting_mp_l2l_stage_1_tuner', + '", "--region=', + location, + '", "--transform_output_path=', + transform_output.uri, + '", "--training_docker_uri=', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125', + '", "--reduce_search_space_mode=', + reduce_search_space_mode, + f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}', + '", "--training_base_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train', + '", "--num_parallel_trial=', + num_parallel_trials, + '", "--single_run_max_secs=', + single_run_max_secs, + '", "--deadline_hours=', + deadline_hours, + '", "--num_selected_trials=', + num_selected_trials, + '", "--lro_job_info=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/lro', + '", "--error_file_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb', + '", "--metadata_path=', + metadata.uri, + '", "--materialized_train_split=', + materialized_train_split.uri, + '", "--materialized_eval_split=', + materialized_eval_split.uri, + '", "--tuning_result_output_path=', + tuning_result_output.uri, + '", "--kms_key_name=', + encryption_spec_key_name, + '", "--gcp_resources_path=', + gcp_resources, + '", "--use_json=true', + '", "--log_level=ERROR', + '", "--executor_input={{$.json_escape[1]}}"]}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py new file mode 100644 index 0000000000..f33a2be92d --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/forecasting_stage_2_tuner.py @@ -0,0 +1,147 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Forecasting Stage 2 Tuner component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +# pylint: disable=dangerous-default-value,g-bare-generic,g-doc-args,unused-argument +@dsl.container_component +def automl_forecasting_stage_2_tuner( + project: str, + location: str, + root_dir: str, + num_selected_trials: int, + deadline_hours: float, + num_parallel_trials: int, + single_run_max_secs: int, + metadata: Input[Artifact], + transform_output: Input[Artifact], + materialized_train_split: Input[Artifact], + materialized_eval_split: Input[Artifact], + tuning_result_input_path: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + tuning_result_output: Output[Artifact], + worker_pool_specs_override_json: Optional[list] = [], + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Tunes AutoML Forecasting models and selects top trials. + + Args: + project: Project to run stage 2 tuner. + location: Cloud region for running the component: us-central1). + root_dir: The Cloud Storage location to store the output. + worker_pool_specs_override_json: JSON worker pool specs. E.g., [{"machine_spec": {"machine_type": "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": "n1-standard-16"}}] + num_selected_trials: Number of selected trials. The number of weak learners in the final model. + deadline_hours: Number of hours the cross-validation trainer should run. + num_parallel_trials: Number of parallel training trials. + single_run_max_secs: Max number of seconds each training trial runs. + metadata: The forecasting example gen metadata. + transform_output: The transform output artifact. + materialized_train_split: The materialized train split. + materialized_eval_split: The materialized eval split. + encryption_spec_key_name: Customer-managed encryption key. + tuning_result_input_path: Path to the json of hyperparameter tuning results to use when evaluating models. + + Returns: + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + tuning_result_output: The trained (private) model artifact paths and their hyperparameters. + """ + # fmt: on + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "automl-forecasting-stage-2-tuner-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125', + '", "args": ["forecasting_mp_l2l_stage_2_tuner', + '", "--region=', + location, + '", "--transform_output_path=', + transform_output.uri, + '", "--training_docker_uri=', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125', + f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}', + '", "--training_base_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train', + '", "--num_parallel_trial=', + num_parallel_trials, + '", "--single_run_max_secs=', + single_run_max_secs, + '", "--deadline_hours=', + deadline_hours, + '", "--num_selected_trials=', + num_selected_trials, + '", "--lro_job_info=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/lro', + '", "--error_file_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb', + '", "--metadata_path=', + metadata.uri, + '", "--materialized_train_split=', + materialized_train_split.uri, + '", "--materialized_eval_split=', + materialized_eval_split.uri, + '", "--tuning_result_input_path=', + tuning_result_input_path.uri, + '", "--kms_key_name=', + encryption_spec_key_name, + '", "--gcp_resources_path=', + gcp_resources, + '", "--tuning_result_output_path=', + tuning_result_output.uri, + ( + '", "--use_json=true", "--log_level=ERROR",' + ' "--executor_input={{$.json_escape[1]}}"]}}]}}' + ), + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml new file mode 100644 index 0000000000..ea5e0f0de2 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/learn_to_learn_forecasting_pipeline.yaml @@ -0,0 +1,7903 @@ +# PIPELINE DEFINITION +# Name: learn-to-learn-forecasting +# Description: The AutoML Forecasting pipeline. +# Inputs: +# available_at_forecast_columns: list +# context_window: int [Default: 0.0] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# enable_probabilistic_inference: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# evaluated_examples_bigquery_path: str [Default: ''] +# evaluation_batch_explain_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_explain_max_replica_count: int [Default: 22.0] +# evaluation_batch_explain_starting_replica_count: int [Default: 22.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-standard-16'] +# evaluation_batch_predict_max_replica_count: int [Default: 25.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 25.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-16'] +# evaluation_dataflow_max_num_workers: int [Default: 25.0] +# evaluation_dataflow_starting_num_workers: int [Default: 22.0] +# fast_testing: bool [Default: False] +# feature_transform_engine_bigquery_staging_full_dataset_id: str [Default: ''] +# feature_transform_engine_dataflow_disk_size_gb: int [Default: 40.0] +# feature_transform_engine_dataflow_machine_type: str [Default: 'n1-standard-16'] +# feature_transform_engine_dataflow_max_num_workers: int [Default: 10.0] +# forecast_horizon: int [Default: 0.0] +# group_columns: list +# group_temporal_total_weight: float [Default: 0.0] +# group_total_weight: float [Default: 0.0] +# holiday_regions: list +# location: str +# model_description: str [Default: ''] +# model_display_name: str [Default: 'automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'] +# num_selected_trials: int [Default: 10.0] +# optimization_objective: str +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# project: str +# quantiles: list +# root_dir: str +# run_evaluation: bool [Default: False] +# stage_1_num_parallel_trials: int [Default: 35.0] +# stage_1_tuner_worker_pool_specs_override: list +# stage_1_tuning_result_artifact_uri: str [Default: ''] +# stage_2_num_parallel_trials: int [Default: 35.0] +# stage_2_trainer_worker_pool_specs_override: list +# study_spec_parameters_override: list +# target_column: str +# temporal_total_weight: float [Default: 0.0] +# test_fraction: float [Default: -1.0] +# time_column: str +# time_series_attribute_columns: list +# time_series_identifier_columns: list +# timestamp_split_key: str [Default: ''] +# train_budget_milli_node_hours: float +# training_fraction: float [Default: -1.0] +# transformations: dict +# unavailable_at_forecast_columns: list +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# window_max_count: int [Default: 0.0] +# window_predefined_column: str [Default: ''] +# window_stride_length: int [Default: 0.0] +# Outputs: +# feature-attribution-2-feature_attributions: system.Metrics +# feature-attribution-feature_attributions: system.Metrics +components: + comp-automl-forecasting-ensemble: + executorLabel: exec-automl-forecasting-ensemble + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-ensemble-2: + executorLabel: exec-automl-forecasting-ensemble-2 + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-1-tuner: + executorLabel: exec-automl-forecasting-stage-1-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the hyperparameter tuning should + + run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the hyperparameter tuning. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run hyperparameter tuning. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "activation","categorical_value_spec": {"values": + + ["tanh"]}}]' + isOptional: true + parameterType: LIST + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-2-tuner: + executorLabel: exec-automl-forecasting-stage-2-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The forecasting example gen + + metadata.' + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Path to the json of hyperparameter + + tuning results to use when evaluating models.' + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: 'Cloud region for running the component: us-central1).' + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model.' + parameterType: NUMBER_INTEGER + project: + description: Project to run stage 2 tuner. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained (private) model artifact paths and their hyperparameters. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-calculate-training-parameters: + executorLabel: exec-calculate-training-parameters + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-calculate-training-parameters-2: + executorLabel: exec-calculate-training-parameters-2 + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-condition-2: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-3 + tasks: + automl-forecasting-ensemble: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble + dependentTasks: + - automl-forecasting-stage-2-tuner + - get-prediction-image-uri + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-2-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble + automl-forecasting-stage-2-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-2-tuner + dependentTasks: + - calculate-training-parameters + - importer + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input_path: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-2-tuner + calculate-training-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: true + selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters + condition-3: + componentRef: + name: comp-condition-3 + dependentTasks: + - automl-forecasting-ensemble + - model-upload + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description + get-prediction-image-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri + inputs: + parameters: + model_type: + runtimeValue: + constant: l2l + taskInfo: + name: get-prediction-image-uri + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: get-hyperparameter-tuning-results + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-forecasting-ensemble + - get-or-create-model-description + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution + tasks: + feature-attribution: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution + dependentTasks: + - model-batch-explanation + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution + finalize-eval-quantile-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters + inputs: + parameters: + quantiles: + componentInputParameter: pipelinechannel--quantiles + taskInfo: + name: finalize-eval-quantile-parameters + get-predictions-column: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column + dependentTasks: + - finalize-eval-quantile-parameters + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column + model-batch-explanation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation-forecasting: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting + dependentTasks: + - finalize-eval-quantile-parameters + - get-predictions-column + - model-batch-predict + - table-to-uri + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - feature-attribution + - model-evaluation-forecasting + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import + table-to-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri + dependentTasks: + - model-batch-predict + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-4: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-5 + tasks: + automl-forecasting-ensemble-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble-2 + dependentTasks: + - automl-forecasting-stage-1-tuner + - get-prediction-image-uri-2 + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-1-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri-2 + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble-2 + automl-forecasting-stage-1-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-1-tuner + dependentTasks: + - calculate-training-parameters-2 + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + runtimeValue: + constant: full + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-1-tuner + calculate-training-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters-2 + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: false + selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters-2 + condition-5: + componentRef: + name: comp-condition-5 + dependentTasks: + - automl-forecasting-ensemble-2 + - model-upload-2 + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--model-upload-2-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-2 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description-2 + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description-2 + get-prediction-image-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri-2 + inputs: + parameters: + model_type: + runtimeValue: + constant: l2l + taskInfo: + name: get-prediction-image-uri-2 + model-upload-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-2 + dependentTasks: + - automl-forecasting-ensemble-2 + - get-or-create-model-description-2 + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description-2 + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-2 + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-5: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-2 + tasks: + feature-attribution-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-2 + dependentTasks: + - model-batch-explanation-2 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-2 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-2 + finalize-eval-quantile-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters-2 + inputs: + parameters: + quantiles: + componentInputParameter: pipelinechannel--quantiles + taskInfo: + name: finalize-eval-quantile-parameters-2 + get-predictions-column-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column-2 + model-batch-explanation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-2 + model-batch-predict-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-2 + model-evaluation-forecasting-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + - get-predictions-column-2 + - model-batch-predict-2 + - table-to-uri-2 + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters-2 + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri-2 + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column-2 + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting-2 + model-evaluation-import-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-2 + dependentTasks: + - feature-attribution-2 + - model-evaluation-forecasting-2 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-2 + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting-2 + model: + componentInputArtifact: pipelinechannel--model-upload-2-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import-2 + table-to-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri-2 + dependentTasks: + - model-batch-predict-2 + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-2-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-4 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-2 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_not_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'true' + condition-4: + componentRef: + name: comp-condition-4 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'false' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + forecasting_context_window: + componentInputParameter: pipelinechannel--context_window + forecasting_forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_holiday_regions: + componentInputParameter: pipelinechannel--holiday_regions + forecasting_predefined_window_column: + componentInputParameter: pipelinechannel--window_predefined_column + forecasting_time_column: + componentInputParameter: pipelinechannel--time_column + forecasting_time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + forecasting_time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + forecasting_unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + forecasting_window_max_count: + componentInputParameter: pipelinechannel--window_max_count + forecasting_window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + location: + componentInputParameter: pipelinechannel--location + model_type: + runtimeValue: + constant: l2l + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + runtimeValue: + constant: time_series + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--transformations + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + string-not-empty: + cachingOptions: + enableCache: true + componentRef: + name: comp-string-not-empty + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: check-if-hyperparameter-tuning-results-are-supplied-by-user + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + context_window: + componentInputParameter: pipelinechannel--context_window + enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_model_type: + runtimeValue: + constant: l2l + forecasting_transformations: + componentInputParameter: pipelinechannel--set-optional-inputs-transformations + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + prediction_type: + runtimeValue: + constant: time_series + quantiles: + componentInputParameter: pipelinechannel--quantiles + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + time_column: + componentInputParameter: pipelinechannel--time_column + time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--available_at_forecast_columns: + parameterType: LIST + pipelinechannel--context_window: + parameterType: NUMBER_INTEGER + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--enable_probabilistic_inference: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_transform_engine_dataflow_machine_type: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--group_columns: + parameterType: LIST + pipelinechannel--group_temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--group_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--holiday_regions: + parameterType: LIST + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-transformations: + parameterType: STRUCT + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_attribute_columns: + parameterType: LIST + pipelinechannel--time_series_identifier_columns: + parameterType: LIST + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transformations: + parameterType: STRUCT + pipelinechannel--unavailable_at_forecast_columns: + parameterType: LIST + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--window_max_count: + parameterType: NUMBER_INTEGER + pipelinechannel--window_predefined_column: + parameterType: STRING + pipelinechannel--window_stride_length: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-attribution: + executorLabel: exec-feature-attribution + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-2: + executorLabel: exec-feature-attribution-2 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-finalize-eval-quantile-parameters: + executorLabel: exec-finalize-eval-quantile-parameters + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-finalize-eval-quantile-parameters-2: + executorLabel: exec-finalize-eval-quantile-parameters-2 + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-get-or-create-model-description: + executorLabel: exec-get-or-create-model-description + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-or-create-model-description-2: + executorLabel: exec-get-or-create-model-description-2 + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri: + executorLabel: exec-get-prediction-image-uri + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri-2: + executorLabel: exec-get-prediction-image-uri-2 + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column: + executorLabel: exec-get-predictions-column + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column-2: + executorLabel: exec-get-predictions-column-2 + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-model-batch-explanation: + executorLabel: exec-model-batch-explanation + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-2: + executorLabel: exec-model-batch-explanation-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-2: + executorLabel: exec-model-batch-predict-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation-forecasting: + executorLabel: exec-model-evaluation-forecasting + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-forecasting-2: + executorLabel: exec-model-evaluation-forecasting-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-2: + executorLabel: exec-model-evaluation-import-2 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-2: + executorLabel: exec-model-upload-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + stats_gen_execution_engine: + description: Execution engine used for stats gen in FTE. + parameterType: STRING + transformations: + description: forecasting transformations to append stats gen engine to. + parameterType: STRUCT + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + transformations: + parameterType: STRUCT + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-string-not-empty: + executorLabel: exec-string-not-empty + inputDefinitions: + parameters: + value: + description: String value to be checked. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-table-to-uri: + executorLabel: exec-table-to-uri + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-table-to-uri-2: + executorLabel: exec-table-to-uri-2 + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-automl-forecasting-ensemble: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-ensemble-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-1-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--use_json=true", "\", \"--log_level=ERROR", "\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-2-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-2-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input_path''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-calculate-training-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-calculate-training-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-feature-attribution: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-2: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-finalize-eval-quantile-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-finalize-eval-quantile-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-get-or-create-model-description: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-or-create-model-description-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-prediction-image-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-prediction-image-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-predictions-column: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-predictions-column-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-importer: + importer: + artifactUri: + runtimeParameter: uri + typeSchema: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + exec-model-batch-explanation: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-forecasting: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-forecasting-2: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-2: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-2: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n stats_gen_execution_engine: str,\n transformations: dict,\n\ + ) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ('transformations', dict),\n ],\n):\n \"\"\"Get the\ + \ data source URI.\n\n Args:\n project: The GCP project that runs the\ + \ pipeline components.\n location: The GCP region that runs the pipeline\ + \ components.\n data_source_csv_filenames: The CSV GCS path when data\ + \ source is CSV.\n data_source_bigquery_table_path: The BigQuery table\ + \ when data source is BQ.\n vertex_dataset: The Vertex dataset when data\ + \ source is Vertex dataset.\n model_display_name: The uploaded model's\ + \ display name.\n stats_gen_execution_engine: Execution engine used for\ + \ stats gen in FTE.\n transformations: forecasting transformations to\ + \ append stats gen engine to.\n\n Returns:\n A named tuple of CSV or\ + \ BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n # TODO(b/261504514) Remove this handling when we use the FTE transform\ + \ config.\n transformations['stats_gen_execution_engine'] = stats_gen_execution_engine\n\ + \n if not model_display_name:\n model_display_name = _DEFAULT_MODEL_DISPLAY_NAME\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n 'transformations',\n ],\n\ + \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ + \ model_display_name,\n transformations,\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-string-not-empty: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _string_not_empty + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _string_not_empty(value: str) -> str:\n \"\"\"Check if the input\ + \ string value is not empty.\n\n Args:\n value: String value to be checked.\n\ + \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ + \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ + \ \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-table-to-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-table-to-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: The AutoML Forecasting pipeline. + name: learn-to-learn-forecasting +root: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--available_at_forecast_columns: + componentInputParameter: available_at_forecast_columns + pipelinechannel--context_window: + componentInputParameter: context_window + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--enable_probabilistic_inference: + componentInputParameter: enable_probabilistic_inference + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: fast_testing + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + componentInputParameter: feature_transform_engine_bigquery_staging_full_dataset_id + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + componentInputParameter: feature_transform_engine_dataflow_disk_size_gb + pipelinechannel--feature_transform_engine_dataflow_machine_type: + componentInputParameter: feature_transform_engine_dataflow_machine_type + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + componentInputParameter: feature_transform_engine_dataflow_max_num_workers + pipelinechannel--forecast_horizon: + componentInputParameter: forecast_horizon + pipelinechannel--group_columns: + componentInputParameter: group_columns + pipelinechannel--group_temporal_total_weight: + componentInputParameter: group_temporal_total_weight + pipelinechannel--group_total_weight: + componentInputParameter: group_total_weight + pipelinechannel--holiday_regions: + componentInputParameter: holiday_regions + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--model_display_name: + componentInputParameter: model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: num_selected_trials + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--quantiles: + componentInputParameter: quantiles + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-transformations: + taskOutputParameter: + outputParameterKey: transformations + producerTask: set-optional-inputs + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: stage_2_trainer_worker_pool_specs_override + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--temporal_total_weight: + componentInputParameter: temporal_total_weight + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--time_column: + componentInputParameter: time_column + pipelinechannel--time_series_attribute_columns: + componentInputParameter: time_series_attribute_columns + pipelinechannel--time_series_identifier_columns: + componentInputParameter: time_series_identifier_columns + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: train_budget_milli_node_hours + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transformations: + componentInputParameter: transformations + pipelinechannel--unavailable_at_forecast_columns: + componentInputParameter: unavailable_at_forecast_columns + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--window_max_count: + componentInputParameter: window_max_count + pipelinechannel--window_predefined_column: + componentInputParameter: window_predefined_column + pipelinechannel--window_stride_length: + componentInputParameter: window_stride_length + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + transformations: + componentInputParameter: transformations + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Vertex Model to upload this model as a version to. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + available_at_forecast_columns: + description: 'The columns that are available at the + + forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: 0.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: The full service account name. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: The dataflow subnetwork. + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: '`True` to enable dataflow public IPs.' + isOptional: true + parameterType: BOOLEAN + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is enabled, the + + model will fit a distribution that captures the uncertainty of a + + prediction. If quantiles are specified, then the quantiles of the + + distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluated_examples_bigquery_path: + defaultValue: '' + description: 'The bigquery dataset to write the + + predicted examples into for evaluation, in the format + + `bq://project.dataset`. Only necessary if evaluation is enabled.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch explain components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_max_replica_count: + defaultValue: 22.0 + description: 'The max number of prediction + + server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_explain_starting_replica_count: + defaultValue: 22.0 + description: 'The initial number of + + prediction server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the batch prediction + + job in evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 25.0 + description: 'The maximum count of replicas + + the batch prediction job can scale to.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 25.0 + description: 'Number of replicas to use + + in the batch prediction cluster at startup time.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: The disk space in GB for dataflow. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the dataflow job in + + evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 25.0 + description: Maximum number of dataflow workers. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 22.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + feature_transform_engine_bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'The full id of + + the feature transform engine staging dataset.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size of the + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_transform_engine_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type of + + the feature transform engine.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_max_num_workers: + defaultValue: 10.0 + description: 'The max number of + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon: + defaultValue: 0.0 + description: The length of the horizon. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + description: 'A list of time series attribute column names that define the + + time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions + + aggregated over both the horizon and time series in the same hierarchy + + group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated over + + time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + holiday_regions: + description: 'The geographical regions where the holiday effect is + + applied in modeling.' + isOptional: true + parameterType: LIST + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_description: + defaultValue: '' + description: Optional description. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + description: Optional display name for model. + isOptional: true + parameterType: STRING + num_selected_trials: + defaultValue: 10.0 + description: Number of selected trails. + isOptional: true + parameterType: NUMBER_INTEGER + optimization_objective: + description: '"minimize-rmse", "minimize-mae", "minimize-rmsle", + + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + + "minimize-quantile-loss".' + parameterType: STRING + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + quantiles: + description: 'Quantiles to use for probabilistic inference. Up to 5 quantiles + + are allowed of values between 0 and 1, exclusive. Represents the quantiles + + to use for that objective. Quantiles must be unique.' + isOptional: true + parameterType: LIST + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: '`True` to evaluate the ensembled model on the test split.' + isOptional: true + parameterType: BOOLEAN + stage_1_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 1. + isOptional: true + parameterType: NUMBER_INTEGER + stage_1_tuner_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 1 tuner worker pool spec.' + isOptional: true + parameterType: LIST + stage_1_tuning_result_artifact_uri: + defaultValue: '' + description: 'The stage 1 tuning result artifact GCS + + URI.' + isOptional: true + parameterType: STRING + stage_2_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stage_2_trainer_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 2 trainer worker pool spec.' + isOptional: true + parameterType: LIST + study_spec_parameters_override: + description: The list for overriding study spec. + isOptional: true + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated + + over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + description: The column that indicates the time. + parameterType: STRING + time_series_attribute_columns: + description: 'The columns that are invariant across the + + same time series.' + isOptional: true + parameterType: LIST + time_series_identifier_columns: + description: 'The columns that distinguish the different + + time series.' + parameterType: LIST + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transformations: + description: 'Dict mapping auto and/or type-resolutions to feature + + columns. The supported types are: auto, categorical, numeric, text, and + + timestamp.' + parameterType: STRUCT + unavailable_at_forecast_columns: + description: 'The columns that are unavailable at the + + forecast time.' + isOptional: true + parameterType: LIST + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + window_max_count: + defaultValue: 0.0 + description: The maximum number of windows that will be generated. + isOptional: true + parameterType: NUMBER_INTEGER + window_predefined_column: + defaultValue: '' + description: The column that indicate the start of each window. + isOptional: true + parameterType: STRING + window_stride_length: + defaultValue: 0.0 + description: The stride length to generate the window. + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml new file mode 100644 index 0000000000..a15f89d4f5 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/sequence_to_sequence_forecasting_pipeline.yaml @@ -0,0 +1,7862 @@ +# PIPELINE DEFINITION +# Name: sequence-to-sequence-forecasting +# Description: The Sequence to Sequence (Seq2Seq) Forecasting pipeline. +# Inputs: +# available_at_forecast_columns: list +# context_window: int [Default: 0.0] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# encryption_spec_key_name: str [Default: ''] +# evaluated_examples_bigquery_path: str [Default: ''] +# evaluation_batch_explain_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_explain_max_replica_count: int [Default: 22.0] +# evaluation_batch_explain_starting_replica_count: int [Default: 22.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-standard-16'] +# evaluation_batch_predict_max_replica_count: int [Default: 25.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 25.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-16'] +# evaluation_dataflow_max_num_workers: int [Default: 25.0] +# evaluation_dataflow_starting_num_workers: int [Default: 22.0] +# fast_testing: bool [Default: False] +# feature_transform_engine_bigquery_staging_full_dataset_id: str [Default: ''] +# feature_transform_engine_dataflow_disk_size_gb: int [Default: 40.0] +# feature_transform_engine_dataflow_machine_type: str [Default: 'n1-standard-16'] +# feature_transform_engine_dataflow_max_num_workers: int [Default: 10.0] +# forecast_horizon: int [Default: 0.0] +# group_columns: list +# group_temporal_total_weight: float [Default: 0.0] +# group_total_weight: float [Default: 0.0] +# holiday_regions: list +# location: str +# model_description: str [Default: ''] +# model_display_name: str [Default: 'automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'] +# num_selected_trials: int [Default: 10.0] +# optimization_objective: str +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# project: str +# root_dir: str +# run_evaluation: bool [Default: False] +# stage_1_num_parallel_trials: int [Default: 35.0] +# stage_1_tuner_worker_pool_specs_override: list +# stage_1_tuning_result_artifact_uri: str [Default: ''] +# stage_2_num_parallel_trials: int [Default: 35.0] +# stage_2_trainer_worker_pool_specs_override: list +# study_spec_parameters_override: list +# target_column: str +# temporal_total_weight: float [Default: 0.0] +# test_fraction: float [Default: -1.0] +# time_column: str +# time_series_attribute_columns: list +# time_series_identifier_columns: list +# timestamp_split_key: str [Default: ''] +# train_budget_milli_node_hours: float +# training_fraction: float [Default: -1.0] +# transformations: dict +# unavailable_at_forecast_columns: list +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# window_max_count: int [Default: 0.0] +# window_predefined_column: str [Default: ''] +# window_stride_length: int [Default: 0.0] +# Outputs: +# feature-attribution-2-feature_attributions: system.Metrics +# feature-attribution-feature_attributions: system.Metrics +components: + comp-automl-forecasting-ensemble: + executorLabel: exec-automl-forecasting-ensemble + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-ensemble-2: + executorLabel: exec-automl-forecasting-ensemble-2 + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-1-tuner: + executorLabel: exec-automl-forecasting-stage-1-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the hyperparameter tuning should + + run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the hyperparameter tuning. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run hyperparameter tuning. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "activation","categorical_value_spec": {"values": + + ["tanh"]}}]' + isOptional: true + parameterType: LIST + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-2-tuner: + executorLabel: exec-automl-forecasting-stage-2-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The forecasting example gen + + metadata.' + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Path to the json of hyperparameter + + tuning results to use when evaluating models.' + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: 'Cloud region for running the component: us-central1).' + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model.' + parameterType: NUMBER_INTEGER + project: + description: Project to run stage 2 tuner. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained (private) model artifact paths and their hyperparameters. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-calculate-training-parameters: + executorLabel: exec-calculate-training-parameters + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-calculate-training-parameters-2: + executorLabel: exec-calculate-training-parameters-2 + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-condition-2: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-3 + tasks: + automl-forecasting-ensemble: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble + dependentTasks: + - automl-forecasting-stage-2-tuner + - get-prediction-image-uri + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-2-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble + automl-forecasting-stage-2-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-2-tuner + dependentTasks: + - calculate-training-parameters + - importer + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input_path: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-2-tuner + calculate-training-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: true + selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters + condition-3: + componentRef: + name: comp-condition-3 + dependentTasks: + - automl-forecasting-ensemble + - model-upload + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description + get-prediction-image-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri + inputs: + parameters: + model_type: + runtimeValue: + constant: seq2seq + taskInfo: + name: get-prediction-image-uri + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: get-hyperparameter-tuning-results + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-forecasting-ensemble + - get-or-create-model-description + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution + tasks: + feature-attribution: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution + dependentTasks: + - model-batch-explanation + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution + finalize-eval-quantile-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters + inputs: + parameters: + quantiles: + runtimeValue: + constant: [] + taskInfo: + name: finalize-eval-quantile-parameters + get-predictions-column: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column + dependentTasks: + - finalize-eval-quantile-parameters + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column + model-batch-explanation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation-forecasting: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting + dependentTasks: + - finalize-eval-quantile-parameters + - get-predictions-column + - model-batch-predict + - table-to-uri + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - feature-attribution + - model-evaluation-forecasting + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import + table-to-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri + dependentTasks: + - model-batch-predict + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-4: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-5 + tasks: + automl-forecasting-ensemble-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble-2 + dependentTasks: + - automl-forecasting-stage-1-tuner + - get-prediction-image-uri-2 + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-1-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri-2 + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble-2 + automl-forecasting-stage-1-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-1-tuner + dependentTasks: + - calculate-training-parameters-2 + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + runtimeValue: + constant: full + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-1-tuner + calculate-training-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters-2 + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: false + selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters-2 + condition-5: + componentRef: + name: comp-condition-5 + dependentTasks: + - automl-forecasting-ensemble-2 + - model-upload-2 + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--model-upload-2-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-2 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description-2 + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description-2 + get-prediction-image-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri-2 + inputs: + parameters: + model_type: + runtimeValue: + constant: seq2seq + taskInfo: + name: get-prediction-image-uri-2 + model-upload-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-2 + dependentTasks: + - automl-forecasting-ensemble-2 + - get-or-create-model-description-2 + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description-2 + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-2 + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-5: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-2 + tasks: + feature-attribution-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-2 + dependentTasks: + - model-batch-explanation-2 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-2 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-2 + finalize-eval-quantile-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters-2 + inputs: + parameters: + quantiles: + runtimeValue: + constant: [] + taskInfo: + name: finalize-eval-quantile-parameters-2 + get-predictions-column-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column-2 + model-batch-explanation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-2 + model-batch-predict-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-2 + model-evaluation-forecasting-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + - get-predictions-column-2 + - model-batch-predict-2 + - table-to-uri-2 + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters-2 + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri-2 + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column-2 + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting-2 + model-evaluation-import-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-2 + dependentTasks: + - feature-attribution-2 + - model-evaluation-forecasting-2 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-2 + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting-2 + model: + componentInputArtifact: pipelinechannel--model-upload-2-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import-2 + table-to-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri-2 + dependentTasks: + - model-batch-predict-2 + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-2-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-4 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-2 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_not_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'true' + condition-4: + componentRef: + name: comp-condition-4 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'false' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + forecasting_context_window: + componentInputParameter: pipelinechannel--context_window + forecasting_forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_holiday_regions: + componentInputParameter: pipelinechannel--holiday_regions + forecasting_predefined_window_column: + componentInputParameter: pipelinechannel--window_predefined_column + forecasting_time_column: + componentInputParameter: pipelinechannel--time_column + forecasting_time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + forecasting_time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + forecasting_unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + forecasting_window_max_count: + componentInputParameter: pipelinechannel--window_max_count + forecasting_window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + location: + componentInputParameter: pipelinechannel--location + model_type: + runtimeValue: + constant: seq2seq + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + runtimeValue: + constant: time_series + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--transformations + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + string-not-empty: + cachingOptions: + enableCache: true + componentRef: + name: comp-string-not-empty + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: check-if-hyperparameter-tuning-results-are-supplied-by-user + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + context_window: + componentInputParameter: pipelinechannel--context_window + enable_probabilistic_inference: + runtimeValue: + constant: false + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_model_type: + runtimeValue: + constant: seq2seq + forecasting_transformations: + componentInputParameter: pipelinechannel--set-optional-inputs-transformations + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + prediction_type: + runtimeValue: + constant: time_series + quantiles: + runtimeValue: + constant: [] + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + time_column: + componentInputParameter: pipelinechannel--time_column + time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--available_at_forecast_columns: + parameterType: LIST + pipelinechannel--context_window: + parameterType: NUMBER_INTEGER + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_transform_engine_dataflow_machine_type: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--group_columns: + parameterType: LIST + pipelinechannel--group_temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--group_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--holiday_regions: + parameterType: LIST + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-transformations: + parameterType: STRUCT + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_attribute_columns: + parameterType: LIST + pipelinechannel--time_series_identifier_columns: + parameterType: LIST + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transformations: + parameterType: STRUCT + pipelinechannel--unavailable_at_forecast_columns: + parameterType: LIST + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--window_max_count: + parameterType: NUMBER_INTEGER + pipelinechannel--window_predefined_column: + parameterType: STRING + pipelinechannel--window_stride_length: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-attribution: + executorLabel: exec-feature-attribution + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-2: + executorLabel: exec-feature-attribution-2 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-finalize-eval-quantile-parameters: + executorLabel: exec-finalize-eval-quantile-parameters + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-finalize-eval-quantile-parameters-2: + executorLabel: exec-finalize-eval-quantile-parameters-2 + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-get-or-create-model-description: + executorLabel: exec-get-or-create-model-description + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-or-create-model-description-2: + executorLabel: exec-get-or-create-model-description-2 + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri: + executorLabel: exec-get-prediction-image-uri + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri-2: + executorLabel: exec-get-prediction-image-uri-2 + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column: + executorLabel: exec-get-predictions-column + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column-2: + executorLabel: exec-get-predictions-column-2 + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-model-batch-explanation: + executorLabel: exec-model-batch-explanation + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-2: + executorLabel: exec-model-batch-explanation-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-2: + executorLabel: exec-model-batch-predict-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation-forecasting: + executorLabel: exec-model-evaluation-forecasting + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-forecasting-2: + executorLabel: exec-model-evaluation-forecasting-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-2: + executorLabel: exec-model-evaluation-import-2 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-2: + executorLabel: exec-model-upload-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + stats_gen_execution_engine: + description: Execution engine used for stats gen in FTE. + parameterType: STRING + transformations: + description: forecasting transformations to append stats gen engine to. + parameterType: STRUCT + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + transformations: + parameterType: STRUCT + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-string-not-empty: + executorLabel: exec-string-not-empty + inputDefinitions: + parameters: + value: + description: String value to be checked. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-table-to-uri: + executorLabel: exec-table-to-uri + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-table-to-uri-2: + executorLabel: exec-table-to-uri-2 + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-automl-forecasting-ensemble: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-ensemble-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-1-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--use_json=true", "\", \"--log_level=ERROR", "\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-2-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-2-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input_path''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-calculate-training-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-calculate-training-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-feature-attribution: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-2: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-finalize-eval-quantile-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-finalize-eval-quantile-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-get-or-create-model-description: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-or-create-model-description-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-prediction-image-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-prediction-image-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-predictions-column: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-predictions-column-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-importer: + importer: + artifactUri: + runtimeParameter: uri + typeSchema: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + exec-model-batch-explanation: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-forecasting: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-forecasting-2: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-2: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-2: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n stats_gen_execution_engine: str,\n transformations: dict,\n\ + ) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ('transformations', dict),\n ],\n):\n \"\"\"Get the\ + \ data source URI.\n\n Args:\n project: The GCP project that runs the\ + \ pipeline components.\n location: The GCP region that runs the pipeline\ + \ components.\n data_source_csv_filenames: The CSV GCS path when data\ + \ source is CSV.\n data_source_bigquery_table_path: The BigQuery table\ + \ when data source is BQ.\n vertex_dataset: The Vertex dataset when data\ + \ source is Vertex dataset.\n model_display_name: The uploaded model's\ + \ display name.\n stats_gen_execution_engine: Execution engine used for\ + \ stats gen in FTE.\n transformations: forecasting transformations to\ + \ append stats gen engine to.\n\n Returns:\n A named tuple of CSV or\ + \ BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n # TODO(b/261504514) Remove this handling when we use the FTE transform\ + \ config.\n transformations['stats_gen_execution_engine'] = stats_gen_execution_engine\n\ + \n if not model_display_name:\n model_display_name = _DEFAULT_MODEL_DISPLAY_NAME\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n 'transformations',\n ],\n\ + \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ + \ model_display_name,\n transformations,\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-string-not-empty: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _string_not_empty + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _string_not_empty(value: str) -> str:\n \"\"\"Check if the input\ + \ string value is not empty.\n\n Args:\n value: String value to be checked.\n\ + \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ + \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ + \ \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-table-to-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-table-to-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: The Sequence to Sequence (Seq2Seq) Forecasting pipeline. + name: sequence-to-sequence-forecasting +root: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--available_at_forecast_columns: + componentInputParameter: available_at_forecast_columns + pipelinechannel--context_window: + componentInputParameter: context_window + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: fast_testing + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + componentInputParameter: feature_transform_engine_bigquery_staging_full_dataset_id + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + componentInputParameter: feature_transform_engine_dataflow_disk_size_gb + pipelinechannel--feature_transform_engine_dataflow_machine_type: + componentInputParameter: feature_transform_engine_dataflow_machine_type + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + componentInputParameter: feature_transform_engine_dataflow_max_num_workers + pipelinechannel--forecast_horizon: + componentInputParameter: forecast_horizon + pipelinechannel--group_columns: + componentInputParameter: group_columns + pipelinechannel--group_temporal_total_weight: + componentInputParameter: group_temporal_total_weight + pipelinechannel--group_total_weight: + componentInputParameter: group_total_weight + pipelinechannel--holiday_regions: + componentInputParameter: holiday_regions + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--model_display_name: + componentInputParameter: model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: num_selected_trials + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-transformations: + taskOutputParameter: + outputParameterKey: transformations + producerTask: set-optional-inputs + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: stage_2_trainer_worker_pool_specs_override + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--temporal_total_weight: + componentInputParameter: temporal_total_weight + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--time_column: + componentInputParameter: time_column + pipelinechannel--time_series_attribute_columns: + componentInputParameter: time_series_attribute_columns + pipelinechannel--time_series_identifier_columns: + componentInputParameter: time_series_identifier_columns + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: train_budget_milli_node_hours + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transformations: + componentInputParameter: transformations + pipelinechannel--unavailable_at_forecast_columns: + componentInputParameter: unavailable_at_forecast_columns + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--window_max_count: + componentInputParameter: window_max_count + pipelinechannel--window_predefined_column: + componentInputParameter: window_predefined_column + pipelinechannel--window_stride_length: + componentInputParameter: window_stride_length + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + transformations: + componentInputParameter: transformations + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Vertex model to upload this model as a version to. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + available_at_forecast_columns: + description: 'The columns that are available at the + + forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: 0.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: The full service account name. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: The dataflow subnetwork. + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: '`True` to enable dataflow public IPs.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluated_examples_bigquery_path: + defaultValue: '' + description: 'The bigquery dataset to write the + + predicted examples into for evaluation, in the format + + `bq://project.dataset`. Only necessary if evaluation is enabled.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch explain components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_max_replica_count: + defaultValue: 22.0 + description: 'The max number of prediction + + server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_explain_starting_replica_count: + defaultValue: 22.0 + description: 'The initial number of + + prediction server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the batch prediction + + job in evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 25.0 + description: 'The maximum count of replicas + + the batch prediction job can scale to.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 25.0 + description: 'Number of replicas to use + + in the batch prediction cluster at startup time.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: The disk space in GB for dataflow. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the dataflow job in + + evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 25.0 + description: Maximum number of dataflow workers. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 22.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + feature_transform_engine_bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'The full id of + + the feature transform engine staging dataset.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size of the + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_transform_engine_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type of + + the feature transform engine.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_max_num_workers: + defaultValue: 10.0 + description: 'The max number of + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon: + defaultValue: 0.0 + description: The length of the horizon. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + description: 'A list of time series attribute column names that define the + + time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions + + aggregated over both the horizon and time series in the same hierarchy + + group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated over + + time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + holiday_regions: + description: 'The geographical regions where the holiday effect is + + applied in modeling.' + isOptional: true + parameterType: LIST + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_description: + defaultValue: '' + description: Optional description. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + description: Optional display name for model. + isOptional: true + parameterType: STRING + num_selected_trials: + defaultValue: 10.0 + description: Number of selected trails. + isOptional: true + parameterType: NUMBER_INTEGER + optimization_objective: + description: '"minimize-rmse", "minimize-mae", "minimize-rmsle", + + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + + "minimize-quantile-loss".' + parameterType: STRING + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: '`True` to evaluate the ensembled model on the test split.' + isOptional: true + parameterType: BOOLEAN + stage_1_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 1. + isOptional: true + parameterType: NUMBER_INTEGER + stage_1_tuner_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 1 tuner worker pool spec.' + isOptional: true + parameterType: LIST + stage_1_tuning_result_artifact_uri: + defaultValue: '' + description: 'The stage 1 tuning result artifact GCS + + URI.' + isOptional: true + parameterType: STRING + stage_2_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stage_2_trainer_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 2 trainer worker pool spec.' + isOptional: true + parameterType: LIST + study_spec_parameters_override: + description: The list for overriding study spec. + isOptional: true + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated + + over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + description: The column that indicates the time. + parameterType: STRING + time_series_attribute_columns: + description: 'The columns that are invariant across the + + same time series.' + isOptional: true + parameterType: LIST + time_series_identifier_columns: + description: 'The columns that distinguish the different + + time series.' + parameterType: LIST + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transformations: + description: 'Dict mapping auto and/or type-resolutions to feature + + columns. The supported types are: auto, categorical, numeric, text, and + + timestamp.' + parameterType: STRUCT + unavailable_at_forecast_columns: + description: 'The columns that are unavailable at the + + forecast time.' + isOptional: true + parameterType: LIST + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + window_max_count: + defaultValue: 0.0 + description: The maximum number of windows that will be generated. + isOptional: true + parameterType: NUMBER_INTEGER + window_predefined_column: + defaultValue: '' + description: The column that indicate the start of each window. + isOptional: true + parameterType: STRING + window_stride_length: + defaultValue: 0.0 + description: The stride length to generate the window. + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml new file mode 100644 index 0000000000..87b9f8f710 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/temporal_fusion_transformer_forecasting_pipeline.yaml @@ -0,0 +1,7848 @@ +# PIPELINE DEFINITION +# Name: temporal-fusion-transformer-forecasting +# Description: The Temporal Fusion Transformer (TFT) Forecasting pipeline. +# Inputs: +# available_at_forecast_columns: list +# context_window: int [Default: 0.0] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# encryption_spec_key_name: str [Default: ''] +# evaluated_examples_bigquery_path: str [Default: ''] +# evaluation_batch_explain_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_explain_max_replica_count: int [Default: 22.0] +# evaluation_batch_explain_starting_replica_count: int [Default: 22.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-standard-16'] +# evaluation_batch_predict_max_replica_count: int [Default: 25.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 25.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-16'] +# evaluation_dataflow_max_num_workers: int [Default: 25.0] +# evaluation_dataflow_starting_num_workers: int [Default: 22.0] +# fast_testing: bool [Default: False] +# feature_transform_engine_bigquery_staging_full_dataset_id: str [Default: ''] +# feature_transform_engine_dataflow_disk_size_gb: int [Default: 40.0] +# feature_transform_engine_dataflow_machine_type: str [Default: 'n1-standard-16'] +# feature_transform_engine_dataflow_max_num_workers: int [Default: 10.0] +# forecast_horizon: int [Default: 0.0] +# group_columns: list +# group_temporal_total_weight: float [Default: 0.0] +# group_total_weight: float [Default: 0.0] +# holiday_regions: list +# location: str +# model_description: str [Default: ''] +# model_display_name: str [Default: 'automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'] +# optimization_objective: str +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# project: str +# root_dir: str +# run_evaluation: bool [Default: False] +# stage_1_num_parallel_trials: int [Default: 35.0] +# stage_1_tuner_worker_pool_specs_override: list +# stage_1_tuning_result_artifact_uri: str [Default: ''] +# stage_2_num_parallel_trials: int [Default: 35.0] +# stage_2_trainer_worker_pool_specs_override: list +# study_spec_parameters_override: list +# target_column: str +# temporal_total_weight: float [Default: 0.0] +# test_fraction: float [Default: -1.0] +# time_column: str +# time_series_attribute_columns: list +# time_series_identifier_columns: list +# timestamp_split_key: str [Default: ''] +# train_budget_milli_node_hours: float +# training_fraction: float [Default: -1.0] +# transformations: dict +# unavailable_at_forecast_columns: list +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# window_max_count: int [Default: 0.0] +# window_predefined_column: str [Default: ''] +# window_stride_length: int [Default: 0.0] +# Outputs: +# feature-attribution-2-feature_attributions: system.Metrics +# feature-attribution-feature_attributions: system.Metrics +components: + comp-automl-forecasting-ensemble: + executorLabel: exec-automl-forecasting-ensemble + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-ensemble-2: + executorLabel: exec-automl-forecasting-ensemble-2 + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-1-tuner: + executorLabel: exec-automl-forecasting-stage-1-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the hyperparameter tuning should + + run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the hyperparameter tuning. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run hyperparameter tuning. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "activation","categorical_value_spec": {"values": + + ["tanh"]}}]' + isOptional: true + parameterType: LIST + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-2-tuner: + executorLabel: exec-automl-forecasting-stage-2-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The forecasting example gen + + metadata.' + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Path to the json of hyperparameter + + tuning results to use when evaluating models.' + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: 'Cloud region for running the component: us-central1).' + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model.' + parameterType: NUMBER_INTEGER + project: + description: Project to run stage 2 tuner. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained (private) model artifact paths and their hyperparameters. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-calculate-training-parameters: + executorLabel: exec-calculate-training-parameters + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-calculate-training-parameters-2: + executorLabel: exec-calculate-training-parameters-2 + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-condition-2: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-3 + tasks: + automl-forecasting-ensemble: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble + dependentTasks: + - automl-forecasting-stage-2-tuner + - get-prediction-image-uri + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-2-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble + automl-forecasting-stage-2-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-2-tuner + dependentTasks: + - calculate-training-parameters + - importer + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input_path: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + runtimeValue: + constant: 1.0 + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-2-tuner + calculate-training-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: true + selected_trials: + runtimeValue: + constant: 1.0 + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters + condition-3: + componentRef: + name: comp-condition-3 + dependentTasks: + - automl-forecasting-ensemble + - model-upload + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description + get-prediction-image-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri + inputs: + parameters: + model_type: + runtimeValue: + constant: tft + taskInfo: + name: get-prediction-image-uri + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: get-hyperparameter-tuning-results + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-forecasting-ensemble + - get-or-create-model-description + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution + tasks: + feature-attribution: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution + dependentTasks: + - model-batch-explanation + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution + finalize-eval-quantile-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters + inputs: + parameters: + quantiles: + runtimeValue: + constant: [] + taskInfo: + name: finalize-eval-quantile-parameters + get-predictions-column: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column + dependentTasks: + - finalize-eval-quantile-parameters + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column + model-batch-explanation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation-forecasting: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting + dependentTasks: + - finalize-eval-quantile-parameters + - get-predictions-column + - model-batch-predict + - table-to-uri + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - feature-attribution + - model-evaluation-forecasting + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import + table-to-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri + dependentTasks: + - model-batch-predict + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-4: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-5 + tasks: + automl-forecasting-ensemble-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble-2 + dependentTasks: + - automl-forecasting-stage-1-tuner + - get-prediction-image-uri-2 + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-1-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri-2 + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble-2 + automl-forecasting-stage-1-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-1-tuner + dependentTasks: + - calculate-training-parameters-2 + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + runtimeValue: + constant: 1.0 + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + runtimeValue: + constant: full + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-1-tuner + calculate-training-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters-2 + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: false + selected_trials: + runtimeValue: + constant: 1.0 + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters-2 + condition-5: + componentRef: + name: comp-condition-5 + dependentTasks: + - automl-forecasting-ensemble-2 + - model-upload-2 + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--model-upload-2-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-2 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description-2 + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description-2 + get-prediction-image-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri-2 + inputs: + parameters: + model_type: + runtimeValue: + constant: tft + taskInfo: + name: get-prediction-image-uri-2 + model-upload-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-2 + dependentTasks: + - automl-forecasting-ensemble-2 + - get-or-create-model-description-2 + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description-2 + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-2 + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-5: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-2 + tasks: + feature-attribution-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-2 + dependentTasks: + - model-batch-explanation-2 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-2 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-2 + finalize-eval-quantile-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters-2 + inputs: + parameters: + quantiles: + runtimeValue: + constant: [] + taskInfo: + name: finalize-eval-quantile-parameters-2 + get-predictions-column-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column-2 + model-batch-explanation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-2 + model-batch-predict-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-2 + model-evaluation-forecasting-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + - get-predictions-column-2 + - model-batch-predict-2 + - table-to-uri-2 + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters-2 + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri-2 + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column-2 + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting-2 + model-evaluation-import-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-2 + dependentTasks: + - feature-attribution-2 + - model-evaluation-forecasting-2 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-2 + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting-2 + model: + componentInputArtifact: pipelinechannel--model-upload-2-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import-2 + table-to-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri-2 + dependentTasks: + - model-batch-predict-2 + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-2-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-4 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-2 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_not_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'true' + condition-4: + componentRef: + name: comp-condition-4 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'false' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + forecasting_context_window: + componentInputParameter: pipelinechannel--context_window + forecasting_forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_holiday_regions: + componentInputParameter: pipelinechannel--holiday_regions + forecasting_predefined_window_column: + componentInputParameter: pipelinechannel--window_predefined_column + forecasting_time_column: + componentInputParameter: pipelinechannel--time_column + forecasting_time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + forecasting_time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + forecasting_unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + forecasting_window_max_count: + componentInputParameter: pipelinechannel--window_max_count + forecasting_window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + location: + componentInputParameter: pipelinechannel--location + model_type: + runtimeValue: + constant: tft + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + runtimeValue: + constant: time_series + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--transformations + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + string-not-empty: + cachingOptions: + enableCache: true + componentRef: + name: comp-string-not-empty + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: check-if-hyperparameter-tuning-results-are-supplied-by-user + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + context_window: + componentInputParameter: pipelinechannel--context_window + enable_probabilistic_inference: + runtimeValue: + constant: false + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_model_type: + runtimeValue: + constant: tft + forecasting_transformations: + componentInputParameter: pipelinechannel--set-optional-inputs-transformations + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + prediction_type: + runtimeValue: + constant: time_series + quantiles: + runtimeValue: + constant: [] + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + time_column: + componentInputParameter: pipelinechannel--time_column + time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--available_at_forecast_columns: + parameterType: LIST + pipelinechannel--context_window: + parameterType: NUMBER_INTEGER + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_transform_engine_dataflow_machine_type: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--group_columns: + parameterType: LIST + pipelinechannel--group_temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--group_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--holiday_regions: + parameterType: LIST + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-transformations: + parameterType: STRUCT + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_attribute_columns: + parameterType: LIST + pipelinechannel--time_series_identifier_columns: + parameterType: LIST + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transformations: + parameterType: STRUCT + pipelinechannel--unavailable_at_forecast_columns: + parameterType: LIST + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--window_max_count: + parameterType: NUMBER_INTEGER + pipelinechannel--window_predefined_column: + parameterType: STRING + pipelinechannel--window_stride_length: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-attribution: + executorLabel: exec-feature-attribution + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-2: + executorLabel: exec-feature-attribution-2 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-finalize-eval-quantile-parameters: + executorLabel: exec-finalize-eval-quantile-parameters + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-finalize-eval-quantile-parameters-2: + executorLabel: exec-finalize-eval-quantile-parameters-2 + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-get-or-create-model-description: + executorLabel: exec-get-or-create-model-description + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-or-create-model-description-2: + executorLabel: exec-get-or-create-model-description-2 + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri: + executorLabel: exec-get-prediction-image-uri + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri-2: + executorLabel: exec-get-prediction-image-uri-2 + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column: + executorLabel: exec-get-predictions-column + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column-2: + executorLabel: exec-get-predictions-column-2 + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-model-batch-explanation: + executorLabel: exec-model-batch-explanation + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-2: + executorLabel: exec-model-batch-explanation-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-2: + executorLabel: exec-model-batch-predict-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation-forecasting: + executorLabel: exec-model-evaluation-forecasting + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-forecasting-2: + executorLabel: exec-model-evaluation-forecasting-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-2: + executorLabel: exec-model-evaluation-import-2 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-2: + executorLabel: exec-model-upload-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + stats_gen_execution_engine: + description: Execution engine used for stats gen in FTE. + parameterType: STRING + transformations: + description: forecasting transformations to append stats gen engine to. + parameterType: STRUCT + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + transformations: + parameterType: STRUCT + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-string-not-empty: + executorLabel: exec-string-not-empty + inputDefinitions: + parameters: + value: + description: String value to be checked. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-table-to-uri: + executorLabel: exec-table-to-uri + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-table-to-uri-2: + executorLabel: exec-table-to-uri-2 + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-automl-forecasting-ensemble: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-ensemble-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-1-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--use_json=true", "\", \"--log_level=ERROR", "\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-2-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-2-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input_path''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-calculate-training-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-calculate-training-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-feature-attribution: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-2: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-finalize-eval-quantile-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-finalize-eval-quantile-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-get-or-create-model-description: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-or-create-model-description-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-prediction-image-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-prediction-image-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-predictions-column: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-predictions-column-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-importer: + importer: + artifactUri: + runtimeParameter: uri + typeSchema: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + exec-model-batch-explanation: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-forecasting: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-forecasting-2: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-2: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-2: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n stats_gen_execution_engine: str,\n transformations: dict,\n\ + ) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ('transformations', dict),\n ],\n):\n \"\"\"Get the\ + \ data source URI.\n\n Args:\n project: The GCP project that runs the\ + \ pipeline components.\n location: The GCP region that runs the pipeline\ + \ components.\n data_source_csv_filenames: The CSV GCS path when data\ + \ source is CSV.\n data_source_bigquery_table_path: The BigQuery table\ + \ when data source is BQ.\n vertex_dataset: The Vertex dataset when data\ + \ source is Vertex dataset.\n model_display_name: The uploaded model's\ + \ display name.\n stats_gen_execution_engine: Execution engine used for\ + \ stats gen in FTE.\n transformations: forecasting transformations to\ + \ append stats gen engine to.\n\n Returns:\n A named tuple of CSV or\ + \ BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n # TODO(b/261504514) Remove this handling when we use the FTE transform\ + \ config.\n transformations['stats_gen_execution_engine'] = stats_gen_execution_engine\n\ + \n if not model_display_name:\n model_display_name = _DEFAULT_MODEL_DISPLAY_NAME\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n 'transformations',\n ],\n\ + \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ + \ model_display_name,\n transformations,\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-string-not-empty: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _string_not_empty + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _string_not_empty(value: str) -> str:\n \"\"\"Check if the input\ + \ string value is not empty.\n\n Args:\n value: String value to be checked.\n\ + \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ + \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ + \ \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-table-to-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-table-to-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: The Temporal Fusion Transformer (TFT) Forecasting pipeline. + name: temporal-fusion-transformer-forecasting +root: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--available_at_forecast_columns: + componentInputParameter: available_at_forecast_columns + pipelinechannel--context_window: + componentInputParameter: context_window + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: fast_testing + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + componentInputParameter: feature_transform_engine_bigquery_staging_full_dataset_id + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + componentInputParameter: feature_transform_engine_dataflow_disk_size_gb + pipelinechannel--feature_transform_engine_dataflow_machine_type: + componentInputParameter: feature_transform_engine_dataflow_machine_type + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + componentInputParameter: feature_transform_engine_dataflow_max_num_workers + pipelinechannel--forecast_horizon: + componentInputParameter: forecast_horizon + pipelinechannel--group_columns: + componentInputParameter: group_columns + pipelinechannel--group_temporal_total_weight: + componentInputParameter: group_temporal_total_weight + pipelinechannel--group_total_weight: + componentInputParameter: group_total_weight + pipelinechannel--holiday_regions: + componentInputParameter: holiday_regions + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--model_display_name: + componentInputParameter: model_display_name + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-transformations: + taskOutputParameter: + outputParameterKey: transformations + producerTask: set-optional-inputs + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: stage_2_trainer_worker_pool_specs_override + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--temporal_total_weight: + componentInputParameter: temporal_total_weight + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--time_column: + componentInputParameter: time_column + pipelinechannel--time_series_attribute_columns: + componentInputParameter: time_series_attribute_columns + pipelinechannel--time_series_identifier_columns: + componentInputParameter: time_series_identifier_columns + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: train_budget_milli_node_hours + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transformations: + componentInputParameter: transformations + pipelinechannel--unavailable_at_forecast_columns: + componentInputParameter: unavailable_at_forecast_columns + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--window_max_count: + componentInputParameter: window_max_count + pipelinechannel--window_predefined_column: + componentInputParameter: window_predefined_column + pipelinechannel--window_stride_length: + componentInputParameter: window_stride_length + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + transformations: + componentInputParameter: transformations + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Optional Vertex Model that this model is a version of. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + available_at_forecast_columns: + description: 'The columns that are available at the + + forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: 0.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: The full service account name. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: The dataflow subnetwork. + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: '`True` to enable dataflow public IPs.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluated_examples_bigquery_path: + defaultValue: '' + description: 'The bigquery dataset to write the + + predicted examples into for evaluation, in the format + + `bq://project.dataset`. Only necessary if evaluation is enabled.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch explain components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_max_replica_count: + defaultValue: 22.0 + description: 'The max number of prediction + + server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_explain_starting_replica_count: + defaultValue: 22.0 + description: 'The initial number of + + prediction server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the batch prediction + + job in evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 25.0 + description: 'The maximum count of replicas + + the batch prediction job can scale to.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 25.0 + description: 'Number of replicas to use + + in the batch prediction cluster at startup time.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: The disk space in GB for dataflow. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the dataflow job in + + evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 25.0 + description: Maximum number of dataflow workers. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 22.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + feature_transform_engine_bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'The full id of + + the feature transform engine staging dataset.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size of the + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_transform_engine_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type of + + the feature transform engine.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_max_num_workers: + defaultValue: 10.0 + description: 'The max number of + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon: + defaultValue: 0.0 + description: The length of the horizon. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + description: 'A list of time series attribute column names that define the + + time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions + + aggregated over both the horizon and time series in the same hierarchy + + group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated over + + time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + holiday_regions: + description: 'The geographical regions where the holiday effect is + + applied in modeling.' + isOptional: true + parameterType: LIST + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_description: + defaultValue: '' + description: Optional description. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + description: Optional display name for model. + isOptional: true + parameterType: STRING + optimization_objective: + description: '"minimize-rmse", "minimize-mae", "minimize-rmsle", + + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + + "minimize-quantile-loss".' + parameterType: STRING + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: '`True` to evaluate the ensembled model on the test split.' + isOptional: true + parameterType: BOOLEAN + stage_1_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 1. + isOptional: true + parameterType: NUMBER_INTEGER + stage_1_tuner_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 1 tuner worker pool spec.' + isOptional: true + parameterType: LIST + stage_1_tuning_result_artifact_uri: + defaultValue: '' + description: 'The stage 1 tuning result artifact GCS + + URI.' + isOptional: true + parameterType: STRING + stage_2_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stage_2_trainer_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 2 trainer worker pool spec.' + isOptional: true + parameterType: LIST + study_spec_parameters_override: + description: The list for overriding study spec. + isOptional: true + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated + + over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + description: The column that indicates the time. + parameterType: STRING + time_series_attribute_columns: + description: 'The columns that are invariant across the + + same time series.' + isOptional: true + parameterType: LIST + time_series_identifier_columns: + description: 'The columns that distinguish the different + + time series.' + parameterType: LIST + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transformations: + description: 'Dict mapping auto and/or type-resolutions to feature + + columns. The supported types are: auto, categorical, numeric, text, and + + timestamp.' + parameterType: STRUCT + unavailable_at_forecast_columns: + description: 'The columns that are unavailable at the + + forecast time.' + isOptional: true + parameterType: LIST + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + window_max_count: + defaultValue: 0.0 + description: The maximum number of windows that will be generated. + isOptional: true + parameterType: NUMBER_INTEGER + window_predefined_column: + defaultValue: '' + description: The column that indicate the start of each window. + isOptional: true + parameterType: STRING + window_stride_length: + defaultValue: 0.0 + description: The stride length to generate the window. + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml new file mode 100644 index 0000000000..50d25236fa --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/time_series_dense_encoder_forecasting_pipeline.yaml @@ -0,0 +1,7903 @@ +# PIPELINE DEFINITION +# Name: time-series-dense-encoder-forecasting +# Description: The Timeseries Dense Encoder (TiDE) Forecasting pipeline. +# Inputs: +# available_at_forecast_columns: list +# context_window: int [Default: 0.0] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# enable_probabilistic_inference: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# evaluated_examples_bigquery_path: str [Default: ''] +# evaluation_batch_explain_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_explain_max_replica_count: int [Default: 22.0] +# evaluation_batch_explain_starting_replica_count: int [Default: 22.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-standard-16'] +# evaluation_batch_predict_max_replica_count: int [Default: 25.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 25.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-16'] +# evaluation_dataflow_max_num_workers: int [Default: 25.0] +# evaluation_dataflow_starting_num_workers: int [Default: 22.0] +# fast_testing: bool [Default: False] +# feature_transform_engine_bigquery_staging_full_dataset_id: str [Default: ''] +# feature_transform_engine_dataflow_disk_size_gb: int [Default: 40.0] +# feature_transform_engine_dataflow_machine_type: str [Default: 'n1-standard-16'] +# feature_transform_engine_dataflow_max_num_workers: int [Default: 10.0] +# forecast_horizon: int [Default: 0.0] +# group_columns: list +# group_temporal_total_weight: float [Default: 0.0] +# group_total_weight: float [Default: 0.0] +# holiday_regions: list +# location: str +# model_description: str [Default: ''] +# model_display_name: str [Default: 'automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}'] +# num_selected_trials: int [Default: 10.0] +# optimization_objective: str +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# project: str +# quantiles: list +# root_dir: str +# run_evaluation: bool [Default: False] +# stage_1_num_parallel_trials: int [Default: 35.0] +# stage_1_tuner_worker_pool_specs_override: list +# stage_1_tuning_result_artifact_uri: str [Default: ''] +# stage_2_num_parallel_trials: int [Default: 35.0] +# stage_2_trainer_worker_pool_specs_override: list +# study_spec_parameters_override: list +# target_column: str +# temporal_total_weight: float [Default: 0.0] +# test_fraction: float [Default: -1.0] +# time_column: str +# time_series_attribute_columns: list +# time_series_identifier_columns: list +# timestamp_split_key: str [Default: ''] +# train_budget_milli_node_hours: float +# training_fraction: float [Default: -1.0] +# transformations: dict +# unavailable_at_forecast_columns: list +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# window_max_count: int [Default: 0.0] +# window_predefined_column: str [Default: ''] +# window_stride_length: int [Default: 0.0] +# Outputs: +# feature-attribution-2-feature_attributions: system.Metrics +# feature-attribution-feature_attributions: system.Metrics +components: + comp-automl-forecasting-ensemble: + executorLabel: exec-automl-forecasting-ensemble + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-ensemble-2: + executorLabel: exec-automl-forecasting-ensemble-2 + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + instance_schema_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The path to the instance schema, + + describing the input data for the tf_model at serving time.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Region to run the job in. + parameterType: STRING + prediction_image_uri: + description: 'URI of the Docker image to be used as the + + container for serving predictions. This URI must identify an image in + + Artifact Registry or Container Registry.' + parameterType: STRING + project: + description: Project to run the job in. + parameterType: STRING + root_dir: + description: The Cloud Storage path to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + example_instance: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: An example instance which may be used as an input for predictions. + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The explanation metadata used by Vertex online and batch explanations + in the format of a KFP Artifact. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: Model information needed to perform batch prediction. + parameters: + explanation_metadata: + description: The explanation metadata used by Vertex online and batch explanations. + parameterType: STRUCT + explanation_parameters: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-1-tuner: + executorLabel: exec-automl-forecasting-stage-1-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the hyperparameter tuning should + + run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the hyperparameter tuning. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run hyperparameter tuning. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "activation","categorical_value_spec": {"values": + + ["tanh"]}}]' + isOptional: true + parameterType: LIST + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-forecasting-stage-2-tuner: + executorLabel: exec-automl-forecasting-stage-2-tuner + inputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The forecasting example gen + + metadata.' + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Path to the json of hyperparameter + + tuning results to use when evaluating models.' + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: 'Cloud region for running the component: us-central1).' + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model.' + parameterType: NUMBER_INTEGER + project: + description: Project to run stage 2 tuner. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained (private) model artifact paths and their hyperparameters. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-calculate-training-parameters: + executorLabel: exec-calculate-training-parameters + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-calculate-training-parameters-2: + executorLabel: exec-calculate-training-parameters-2 + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + selected_trials: + description: Number of trials that should be selected. + parameterType: NUMBER_INTEGER + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-condition-2: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-3 + tasks: + automl-forecasting-ensemble: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble + dependentTasks: + - automl-forecasting-stage-2-tuner + - get-prediction-image-uri + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-2-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble + automl-forecasting-stage-2-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-2-tuner + dependentTasks: + - calculate-training-parameters + - importer + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input_path: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-2-tuner + calculate-training-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: true + selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters + condition-3: + componentRef: + name: comp-condition-3 + dependentTasks: + - automl-forecasting-ensemble + - model-upload + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description + get-prediction-image-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri + inputs: + parameters: + model_type: + runtimeValue: + constant: tide + taskInfo: + name: get-prediction-image-uri + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: get-hyperparameter-tuning-results + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-forecasting-ensemble + - get-or-create-model-description + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution + tasks: + feature-attribution: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution + dependentTasks: + - model-batch-explanation + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution + finalize-eval-quantile-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters + inputs: + parameters: + quantiles: + componentInputParameter: pipelinechannel--quantiles + taskInfo: + name: finalize-eval-quantile-parameters + get-predictions-column: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column + dependentTasks: + - finalize-eval-quantile-parameters + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column + model-batch-explanation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation-forecasting: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting + dependentTasks: + - finalize-eval-quantile-parameters + - get-predictions-column + - model-batch-predict + - table-to-uri + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - feature-attribution + - model-evaluation-forecasting + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import + table-to-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri + dependentTasks: + - model-batch-predict + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-4: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-5 + tasks: + automl-forecasting-ensemble-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-ensemble-2 + dependentTasks: + - automl-forecasting-stage-1-tuner + - get-prediction-image-uri-2 + inputs: + artifacts: + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-instance_baseline + instance_schema_path: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-forecasting-stage-1-tuner + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_image_uri: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-image-uri-2 + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-forecasting-ensemble-2 + automl-forecasting-stage-1-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-forecasting-stage-1-tuner + dependentTasks: + - calculate-training-parameters-2 + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-metadata + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + runtimeValue: + constant: full + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-forecasting-stage-1-tuner + calculate-training-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters-2 + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: false + selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters-2 + condition-5: + componentRef: + name: comp-condition-5 + dependentTasks: + - automl-forecasting-ensemble-2 + - model-upload-2 + inputs: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--model-upload-2-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-2 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: should_run_model_evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + get-or-create-model-description-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-or-create-model-description-2 + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + original_description: + componentInputParameter: pipelinechannel--model_description + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-or-create-model-description-2 + get-prediction-image-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-image-uri-2 + inputs: + parameters: + model_type: + runtimeValue: + constant: tide + taskInfo: + name: get-prediction-image-uri-2 + model-upload-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-2 + dependentTasks: + - automl-forecasting-ensemble-2 + - get-or-create-model-description-2 + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-forecasting-ensemble-2 + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-forecasting-ensemble-2 + parameters: + description: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-or-create-model-description-2 + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-forecasting-ensemble-2 + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-2 + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-5: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-2 + tasks: + feature-attribution-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-2 + dependentTasks: + - model-batch-explanation-2 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-2 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + runtimeValue: + constant: forecasting + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-2 + finalize-eval-quantile-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-finalize-eval-quantile-parameters-2 + inputs: + parameters: + quantiles: + componentInputParameter: pipelinechannel--quantiles + taskInfo: + name: finalize-eval-quantile-parameters-2 + get-predictions-column-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-predictions-column-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + inputs: + parameters: + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: get-predictions-column-2 + model-batch-explanation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-2 + model-batch-predict-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + generate_explanation: + runtimeValue: + constant: false + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-forecasting-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-2 + model-evaluation-forecasting-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-forecasting-2 + dependentTasks: + - finalize-eval-quantile-parameters-2 + - get-predictions-column-2 + - model-batch-predict-2 + - table-to-uri-2 + inputs: + artifacts: + predictions_bigquery_source: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_quantiles: + taskOutputParameter: + outputParameterKey: quantiles + producerTask: finalize-eval-quantile-parameters-2 + forecasting_type: + taskOutputParameter: + outputParameterKey: forecasting_type + producerTask: finalize-eval-quantile-parameters-2 + ground_truth_bigquery_source: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri-2 + ground_truth_format: + runtimeValue: + constant: bigquery + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-predictions-column-2 + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_field_name: + runtimeValue: + constant: HORIZON__{{$.inputs.parameters['pipelinechannel--target_column']}} + taskInfo: + name: model-evaluation-forecasting-2 + model-evaluation-import-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-2 + dependentTasks: + - feature-attribution-2 + - model-evaluation-forecasting-2 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-2 + forecasting_metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-forecasting-2 + model: + componentInputArtifact: pipelinechannel--model-upload-2-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: Vertex Forecasting pipeline + problem_type: + runtimeValue: + constant: forecasting + taskInfo: + name: model-evaluation-import-2 + table-to-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri-2 + dependentTasks: + - model-batch-predict-2 + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict-2 + parameters: + use_bq_prefix: + runtimeValue: + constant: true + taskInfo: + name: table-to-uri-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-forecasting-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-forecasting-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-2-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-forecasting-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-4 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-2 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_2_trainer_worker_pool_specs_override + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_not_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'true' + condition-4: + componentRef: + name: comp-condition-4 + dependentTasks: + - feature-transform-engine + - split-materialized-data + - string-not-empty + - training-configurator-and-validator + inputs: + artifacts: + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + pipelinechannel--training-configurator-and-validator-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + pipelinechannel--training-configurator-and-validator-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: pipelinechannel--evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: pipelinechannel--num_selected_trials + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'false' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecasting_available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + forecasting_context_window: + componentInputParameter: pipelinechannel--context_window + forecasting_forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_holiday_regions: + componentInputParameter: pipelinechannel--holiday_regions + forecasting_predefined_window_column: + componentInputParameter: pipelinechannel--window_predefined_column + forecasting_time_column: + componentInputParameter: pipelinechannel--time_column + forecasting_time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + forecasting_time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + forecasting_unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + forecasting_window_max_count: + componentInputParameter: pipelinechannel--window_max_count + forecasting_window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + location: + componentInputParameter: pipelinechannel--location + model_type: + runtimeValue: + constant: tide + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + runtimeValue: + constant: time_series + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--transformations + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + string-not-empty: + cachingOptions: + enableCache: true + componentRef: + name: comp-string-not-empty + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: check-if-hyperparameter-tuning-results-are-supplied-by-user + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + available_at_forecast_columns: + componentInputParameter: pipelinechannel--available_at_forecast_columns + context_window: + componentInputParameter: pipelinechannel--context_window + enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_model_type: + runtimeValue: + constant: tide + forecasting_transformations: + componentInputParameter: pipelinechannel--set-optional-inputs-transformations + group_columns: + componentInputParameter: pipelinechannel--group_columns + group_temporal_total_weight: + componentInputParameter: pipelinechannel--group_temporal_total_weight + group_total_weight: + componentInputParameter: pipelinechannel--group_total_weight + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + prediction_type: + runtimeValue: + constant: time_series + quantiles: + componentInputParameter: pipelinechannel--quantiles + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + temporal_total_weight: + componentInputParameter: pipelinechannel--temporal_total_weight + time_column: + componentInputParameter: pipelinechannel--time_column + time_series_attribute_columns: + componentInputParameter: pipelinechannel--time_series_attribute_columns + time_series_identifier_columns: + componentInputParameter: pipelinechannel--time_series_identifier_columns + unavailable_at_forecast_columns: + componentInputParameter: pipelinechannel--unavailable_at_forecast_columns + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--available_at_forecast_columns: + parameterType: LIST + pipelinechannel--context_window: + parameterType: NUMBER_INTEGER + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--enable_probabilistic_inference: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluated_examples_bigquery_path: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_transform_engine_dataflow_machine_type: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--group_columns: + parameterType: LIST + pipelinechannel--group_temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--group_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--holiday_regions: + parameterType: LIST + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-transformations: + parameterType: STRUCT + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--temporal_total_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_attribute_columns: + parameterType: LIST + pipelinechannel--time_series_identifier_columns: + parameterType: LIST + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transformations: + parameterType: STRUCT + pipelinechannel--unavailable_at_forecast_columns: + parameterType: LIST + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--window_max_count: + parameterType: NUMBER_INTEGER + pipelinechannel--window_predefined_column: + parameterType: STRING + pipelinechannel--window_stride_length: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-attribution: + executorLabel: exec-feature-attribution + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-2: + executorLabel: exec-feature-attribution-2 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-finalize-eval-quantile-parameters: + executorLabel: exec-finalize-eval-quantile-parameters + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-finalize-eval-quantile-parameters-2: + executorLabel: exec-finalize-eval-quantile-parameters-2 + inputDefinitions: + parameters: + quantiles: + isOptional: true + parameterType: LIST + outputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + quantiles: + parameterType: LIST + comp-get-or-create-model-description: + executorLabel: exec-get-or-create-model-description + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-or-create-model-description-2: + executorLabel: exec-get-or-create-model-description-2 + inputDefinitions: + parameters: + location: + parameterType: STRING + original_description: + defaultValue: '' + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri: + executorLabel: exec-get-prediction-image-uri + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-prediction-image-uri-2: + executorLabel: exec-get-prediction-image-uri-2 + inputDefinitions: + parameters: + model_type: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column: + executorLabel: exec-get-predictions-column + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-predictions-column-2: + executorLabel: exec-get-predictions-column-2 + inputDefinitions: + parameters: + forecasting_type: + parameterType: STRING + target_column: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-model-batch-explanation: + executorLabel: exec-model-batch-explanation + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-2: + executorLabel: exec-model-batch-explanation-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-2: + executorLabel: exec-model-batch-predict-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation-forecasting: + executorLabel: exec-model-evaluation-forecasting + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-forecasting-2: + executorLabel: exec-model-evaluation-forecasting-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + forecasting_quantiles: + defaultValue: + - 0.5 + isOptional: true + parameterType: LIST + forecasting_type: + defaultValue: point + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + point_evaluation_quantile: + defaultValue: 0.5 + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + target_field_name: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-2: + executorLabel: exec-model-evaluation-import-2 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-2: + executorLabel: exec-model-upload-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + stats_gen_execution_engine: + description: Execution engine used for stats gen in FTE. + parameterType: STRING + transformations: + description: forecasting transformations to append stats gen engine to. + parameterType: STRUCT + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + transformations: + parameterType: STRUCT + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-string-not-empty: + executorLabel: exec-string-not-empty + inputDefinitions: + parameters: + value: + description: String value to be checked. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-table-to-uri: + executorLabel: exec-table-to-uri + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-table-to-uri-2: + executorLabel: exec-table-to-uri-2 + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-automl-forecasting-ensemble: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-ensemble-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"display_name": "automl-forecasting-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "encryption_spec": {"kms_key_name": "{{$.inputs.parameters[''encryption_spec_key_name'']}}"}, + "job_spec": {"worker_pool_specs": [{"replica_count": 1, "machine_spec": + {"machine_type": "n1-highmem-8"}, "container_spec": {"image_uri": "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "args": ["forecasting_mp_ensemble", "--transform_output_path={{$.inputs.artifacts[''transform_output''].uri}}", + "--error_file_path={{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "--metadata_path={{$.inputs.artifacts[''metadata''].uri}}", "--tuning_result_input_path={{$.inputs.artifacts[''tuning_result_input''].uri}}", + "--instance_baseline_path={{$.inputs.artifacts[''instance_baseline''].uri}}", + "--instance_schema_path={{$.inputs.artifacts[''instance_schema_path''].uri}}", + "--prediction_docker_uri={{$.inputs.parameters[''prediction_image_uri'']}}", + "--model_relative_output_path={{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model", + "--explanation_metadata_path={{$.outputs.parameters[''explanation_metadata''].output_file}},{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "--explanation_parameters_path={{$.outputs.parameters[''explanation_parameters''].output_file}}", + "--model_architecture_path={{$.outputs.artifacts[''model_architecture''].uri}}", + "--example_instance_path={{$.outputs.artifacts[''example_instance''].uri}}", + "--use_json=true", "--executor_input={{$.json_escape[1]}}"]}}]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-1-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_1_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--use_json=true", "\", \"--log_level=ERROR", "\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-forecasting-stage-2-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-forecasting-stage-2-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"args\": [\"forecasting_mp_l2l_stage_2_tuner", "\", \"--region=", + "{{$.inputs.parameters[''location'']}}", "\", \"--transform_output_path=", + "{{$.inputs.artifacts[''transform_output''].uri}}", "\", \"--training_docker_uri=", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/forecasting-training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}", "\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train", + "\", \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro", + "\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--materialized_train_split=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input_path''].uri}}", + "\", \"--kms_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--tuning_result_output_path=", "{{$.outputs.artifacts[''tuning_result_output''].uri}}", + "\", \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-calculate-training-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-calculate-training-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n selected_trials: int,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_single_run_max_secs',\ + \ int),\n ('stage_2_deadline_hours', float),\n ('stage_2_single_run_max_secs',\ + \ int),\n ],\n):\n \"\"\"Calculates training parameters.\n\n Args:\n\ + \ stage_1_num_parallel_trials: Number of parallel trails for stage 1.\n\ + \ train_budget_milli_node_hours: The train budget of creating this model,\n\ + \ expressed in milli node hours i.e. 1,000 value in this field means\ + \ 1 node\n hour.\n stage_2_num_parallel_trials: Number of parallel\ + \ trails for stage 2.\n selected_trials: Number of trials that should\ + \ be selected.\n is_skip_architecture_search: If component is being called\ + \ in the\n skip_architecture_search pipeline.\n fast_testing: Internal\ + \ flag used for presubmit tests.\n\n Returns:\n stage_1_deadline_hours:\ + \ Maximum number of hours to run stage 1.\n stage_1_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 1\n training\ + \ trial.\n stage_2_deadline_hours: Maximum number of hours to run stage\ + \ 2.\n stage_2_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 2\n training trial.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n stage_1_deadline_hours = -1.0\n stage_1_single_run_max_secs = -1\n\ + \ stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs = -1\n\n\ + \ if is_skip_architecture_search:\n stage_2_deadline_hours = train_budget_milli_node_hours\ + \ / 1000.0\n rounds = math.ceil(selected_trials / stage_2_num_parallel_trials)\n\ + \ stage_2_single_run_max_secs = int(\n stage_2_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n else:\n stage_1_deadline_hours =\ + \ train_budget_milli_node_hours / 1000.0\n rounds = math.ceil(100 / stage_1_num_parallel_trials)\n\ + \ stage_1_single_run_max_secs = int(\n stage_1_deadline_hours\ + \ * 3600.0 / 1.3 / rounds\n )\n if fast_testing:\n stage_1_deadline_hours\ + \ = 0.2\n stage_1_single_run_max_secs = 1\n stage_2_deadline_hours\ + \ = 0.2\n stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_single_run_max_secs',\n 'stage_2_deadline_hours',\n\ + \ 'stage_2_single_run_max_secs',\n ],\n )(\n stage_1_deadline_hours,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n )\n\n" + image: python:3.7 + exec-feature-attribution: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-2: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-finalize-eval-quantile-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-finalize-eval-quantile-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - finalize_eval_quantile_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef finalize_eval_quantile_parameters(\n quantiles: Optional[list]\ + \ = None, # pylint: disable=g-bare-generic\n) -> NamedTuple('Outputs',\ + \ [('forecasting_type', str), ('quantiles', list)]):\n \"\"\"Infers quantile-specific\ + \ evaluation parameters.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ if not quantiles or quantiles == '[]':\n quantiles = []\n forecasting_type\ + \ = 'point'\n else:\n forecasting_type = 'quantile'\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n (\n 'forecasting_type',\n 'quantiles',\n\ + \ ),\n )(forecasting_type, quantiles)\n\n" + image: python:3.7 + exec-get-or-create-model-description: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-or-create-model-description-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_or_create_model_description + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_or_create_model_description(\n location: str,\n project:\ + \ str,\n original_description: str = '',\n) -> str:\n \"\"\"Creates\ + \ a useful model description if one is not provided.\"\"\"\n # Note: {{$.pipeline_job_name}}\ + \ is dsl.PIPELINE_JOB_NAME_PLACEHOLDER, though\n # at compile time the\ + \ actual template format doesn't get injected since\n # the Python isn't\ + \ interpreted yet, so we have to hardcode the value.\n pipeline_url = 'https://console.cloud.google.com/vertex-ai/locations/{location}/pipelines/runs/{{$.pipeline_job_name}}?project={project}'.format(\n\ + \ location=location, project=project\n )\n if original_description:\n\ + \ return f'{original_description} From: {pipeline_url}'\n\n # The pipeline\ + \ url contains KFP placeholders injected at runtime.\n return f'Vertex\ + \ forecasting model trained in the pipeline: {pipeline_url}'\n\n" + image: python:3.7 + exec-get-prediction-image-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-prediction-image-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_image_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_image_uri(model_type: str) -> str:\n \"\"\"\ + Returns the prediction image corresponding to the given model type.\"\"\"\ + \n # Keys come from AutoMlTimeSeriesForecastingTrainSpec.\n # The URIs\ + \ must be hardcoded without any breaks in the code so string\n # replacement\ + \ will work correctly.\n images = {\n 'l2l': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-l2l:20231002_0125',\n\ + \ 'seq2seq': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-seq2seq:20231002_0125',\n\ + \ 'tft': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tft:20231002_0125',\n\ + \ 'tide': 'us-docker.pkg.dev/vertex-ai/automl-tabular/forecasting-prediction-server-tide:20231002_0125',\n\ + \ }\n if model_type not in images:\n raise ValueError(\n f'Invalid\ + \ forecasting model type: {model_type}. Valid options are: '\n f'{images.keys()}.'\n\ + \ )\n return images[model_type]\n\n" + image: python:3.7 + exec-get-predictions-column: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-predictions-column-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_predictions_column + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_predictions_column(forecasting_type: str, target_column:\ + \ str) -> str:\n \"\"\"Generates the BP output's target column name.\"\"\ + \"\n if forecasting_type == 'quantile':\n return f'predicted_{target_column}.quantile_predictions'\n\ + \ return f'predicted_{target_column}.value'\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-importer: + importer: + artifactUri: + runtimeParameter: uri + typeSchema: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + exec-model-batch-explanation: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-forecasting: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-forecasting-2: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - forecasting + - --forecasting_type + - '{{$.inputs.parameters[''forecasting_type'']}}' + - --forecasting_quantiles + - '{{$.inputs.parameters[''forecasting_quantiles'']}}' + - --point_evaluation_quantile + - '{{$.inputs.parameters[''point_evaluation_quantile'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + "bq://{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}.{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --target_field_name + - instance.{{$.inputs.parameters['target_field_name']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-2: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-2: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n stats_gen_execution_engine: str,\n transformations: dict,\n\ + ) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ('transformations', dict),\n ],\n):\n \"\"\"Get the\ + \ data source URI.\n\n Args:\n project: The GCP project that runs the\ + \ pipeline components.\n location: The GCP region that runs the pipeline\ + \ components.\n data_source_csv_filenames: The CSV GCS path when data\ + \ source is CSV.\n data_source_bigquery_table_path: The BigQuery table\ + \ when data source is BQ.\n vertex_dataset: The Vertex dataset when data\ + \ source is Vertex dataset.\n model_display_name: The uploaded model's\ + \ display name.\n stats_gen_execution_engine: Execution engine used for\ + \ stats gen in FTE.\n transformations: forecasting transformations to\ + \ append stats gen engine to.\n\n Returns:\n A named tuple of CSV or\ + \ BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n # TODO(b/261504514) Remove this handling when we use the FTE transform\ + \ config.\n transformations['stats_gen_execution_engine'] = stats_gen_execution_engine\n\ + \n if not model_display_name:\n model_display_name = _DEFAULT_MODEL_DISPLAY_NAME\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n 'transformations',\n ],\n\ + \ )(\n data_source_csv_filenames,\n data_source_bigquery_table_path,\n\ + \ model_display_name,\n transformations,\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-string-not-empty: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _string_not_empty + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _string_not_empty(value: str) -> str:\n \"\"\"Check if the input\ + \ string value is not empty.\n\n Args:\n value: String value to be checked.\n\ + \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ + \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ + \ \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-table-to-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-table-to-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: The Timeseries Dense Encoder (TiDE) Forecasting pipeline. + name: time-series-dense-encoder-forecasting +root: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--available_at_forecast_columns: + componentInputParameter: available_at_forecast_columns + pipelinechannel--context_window: + componentInputParameter: context_window + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--enable_probabilistic_inference: + componentInputParameter: enable_probabilistic_inference + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluated_examples_bigquery_path: + componentInputParameter: evaluated_examples_bigquery_path + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--fast_testing: + componentInputParameter: fast_testing + pipelinechannel--feature_transform_engine_bigquery_staging_full_dataset_id: + componentInputParameter: feature_transform_engine_bigquery_staging_full_dataset_id + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + componentInputParameter: feature_transform_engine_dataflow_disk_size_gb + pipelinechannel--feature_transform_engine_dataflow_machine_type: + componentInputParameter: feature_transform_engine_dataflow_machine_type + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + componentInputParameter: feature_transform_engine_dataflow_max_num_workers + pipelinechannel--forecast_horizon: + componentInputParameter: forecast_horizon + pipelinechannel--group_columns: + componentInputParameter: group_columns + pipelinechannel--group_temporal_total_weight: + componentInputParameter: group_temporal_total_weight + pipelinechannel--group_total_weight: + componentInputParameter: group_total_weight + pipelinechannel--holiday_regions: + componentInputParameter: holiday_regions + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--model_display_name: + componentInputParameter: model_display_name + pipelinechannel--num_selected_trials: + componentInputParameter: num_selected_trials + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--quantiles: + componentInputParameter: quantiles + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-transformations: + taskOutputParameter: + outputParameterKey: transformations + producerTask: set-optional-inputs + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: stage_2_num_parallel_trials + pipelinechannel--stage_2_trainer_worker_pool_specs_override: + componentInputParameter: stage_2_trainer_worker_pool_specs_override + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--temporal_total_weight: + componentInputParameter: temporal_total_weight + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--time_column: + componentInputParameter: time_column + pipelinechannel--time_series_attribute_columns: + componentInputParameter: time_series_attribute_columns + pipelinechannel--time_series_identifier_columns: + componentInputParameter: time_series_identifier_columns + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: train_budget_milli_node_hours + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transformations: + componentInputParameter: transformations + pipelinechannel--unavailable_at_forecast_columns: + componentInputParameter: unavailable_at_forecast_columns + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--window_max_count: + componentInputParameter: window_max_count + pipelinechannel--window_predefined_column: + componentInputParameter: window_predefined_column + pipelinechannel--window_stride_length: + componentInputParameter: window_stride_length + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + stats_gen_execution_engine: + runtimeValue: + constant: bigquery + transformations: + componentInputParameter: transformations + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Vertex model to upload the model as a version to. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + available_at_forecast_columns: + description: 'The columns that are available at the + + forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: 0.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: The full service account name. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: The dataflow subnetwork. + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: '`True` to enable dataflow public IPs.' + isOptional: true + parameterType: BOOLEAN + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is enabled, the + + model will fit a distribution that captures the uncertainty of a + + prediction. If quantiles are specified, then the quantiles of the + + distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluated_examples_bigquery_path: + defaultValue: '' + description: 'The bigquery dataset to write the + + predicted examples into for evaluation, in the format + + `bq://project.dataset`. Only necessary if evaluation is enabled.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch explain components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_max_replica_count: + defaultValue: 22.0 + description: 'The max number of prediction + + server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_explain_starting_replica_count: + defaultValue: 22.0 + description: 'The initial number of + + prediction server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the batch prediction + + job in evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 25.0 + description: 'The maximum count of replicas + + the batch prediction job can scale to.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 25.0 + description: 'Number of replicas to use + + in the batch prediction cluster at startup time.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: The disk space in GB for dataflow. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'Machine type for the dataflow job in + + evaluation, such as ''n1-standard-16''.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 25.0 + description: Maximum number of dataflow workers. + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 22.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + feature_transform_engine_bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'The full id of + + the feature transform engine staging dataset.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size of the + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_transform_engine_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type of + + the feature transform engine.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_max_num_workers: + defaultValue: 10.0 + description: 'The max number of + + dataflow workers of the feature transform engine.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon: + defaultValue: 0.0 + description: The length of the horizon. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + description: 'A list of time series attribute column names that define the + + time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions + + aggregated over both the horizon and time series in the same hierarchy + + group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated over + + time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + holiday_regions: + description: 'The geographical regions where the holiday effect is + + applied in modeling.' + isOptional: true + parameterType: LIST + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_description: + defaultValue: '' + description: Optional description. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: automl-forecasting-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + description: Optional display name for model. + isOptional: true + parameterType: STRING + num_selected_trials: + defaultValue: 10.0 + description: Number of selected trails. + isOptional: true + parameterType: NUMBER_INTEGER + optimization_objective: + description: '"minimize-rmse", "minimize-mae", "minimize-rmsle", + + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + + "minimize-quantile-loss".' + parameterType: STRING + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + quantiles: + description: 'Quantiles to use for probabilistic inference. Up to 5 quantiles + + are allowed of values between 0 and 1, exclusive. Represents the quantiles + + to use for that objective. Quantiles must be unique.' + isOptional: true + parameterType: LIST + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: '`True` to evaluate the ensembled model on the test split.' + isOptional: true + parameterType: BOOLEAN + stage_1_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 1. + isOptional: true + parameterType: NUMBER_INTEGER + stage_1_tuner_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 1 tuner worker pool spec.' + isOptional: true + parameterType: LIST + stage_1_tuning_result_artifact_uri: + defaultValue: '' + description: 'The stage 1 tuning result artifact GCS + + URI.' + isOptional: true + parameterType: STRING + stage_2_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stage_2_trainer_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 2 trainer worker pool spec.' + isOptional: true + parameterType: LIST + study_spec_parameters_override: + description: The list for overriding study spec. + isOptional: true + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for predictions aggregated + + over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + description: The column that indicates the time. + parameterType: STRING + time_series_attribute_columns: + description: 'The columns that are invariant across the + + same time series.' + isOptional: true + parameterType: LIST + time_series_identifier_columns: + description: 'The columns that distinguish the different + + time series.' + parameterType: LIST + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transformations: + description: 'Dict mapping auto and/or type-resolutions to feature + + columns. The supported types are: auto, categorical, numeric, text, and + + timestamp.' + parameterType: STRUCT + unavailable_at_forecast_columns: + description: 'The columns that are unavailable at the + + forecast time.' + isOptional: true + parameterType: LIST + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + window_max_count: + defaultValue: 0.0 + description: The maximum number of windows that will be generated. + isOptional: true + parameterType: NUMBER_INTEGER + window_predefined_column: + defaultValue: '' + description: The column that indicate the start of each window. + isOptional: true + parameterType: STRING + window_stride_length: + defaultValue: 0.0 + description: The stride length to generate the window. + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/utils.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/utils.py new file mode 100644 index 0000000000..22b4a96eb9 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/forecasting/utils.py @@ -0,0 +1,1037 @@ +"""Util functions for Vertex Forecasting pipelines.""" + +import logging +import os +import pathlib +from typing import Any, Dict, FrozenSet, List, Optional, Tuple + +_GCPC_FORECASTING_PATH = pathlib.Path(__file__).parent.resolve() + +_RETAIL_MODEL_DISABLED_OPTIONS = frozenset([ + 'quantiles', + 'enable_probabilistic_inference', +]) + + +def _get_base_forecasting_parameters( + *, + project: str, + location: str, + root_dir: str, + target_column: str, + optimization_objective: str, + transformations: Dict[str, List[str]], + train_budget_milli_node_hours: float, + time_column: str, + time_series_identifier_columns: List[str], + time_series_identifier_column: Optional[str] = None, + time_series_attribute_columns: Optional[List[str]] = None, + available_at_forecast_columns: Optional[List[str]] = None, + unavailable_at_forecast_columns: Optional[List[str]] = None, + forecast_horizon: Optional[int] = None, + context_window: Optional[int] = None, + evaluated_examples_bigquery_path: Optional[str] = None, + window_predefined_column: Optional[str] = None, + window_stride_length: Optional[int] = None, + window_max_count: Optional[int] = None, + holiday_regions: Optional[List[str]] = None, + stage_1_num_parallel_trials: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + stage_2_num_parallel_trials: Optional[int] = None, + num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + dataflow_service_account: Optional[str] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + feature_transform_engine_bigquery_staging_full_dataset_id: str = '', + feature_transform_engine_dataflow_machine_type: str = 'n1-standard-16', + feature_transform_engine_dataflow_max_num_workers: int = 10, + feature_transform_engine_dataflow_disk_size_gb: int = 40, + evaluation_batch_predict_machine_type: str = 'n1-standard-16', + evaluation_batch_predict_starting_replica_count: int = 25, + evaluation_batch_predict_max_replica_count: int = 25, + evaluation_dataflow_machine_type: str = 'n1-standard-16', + evaluation_dataflow_max_num_workers: int = 25, + evaluation_dataflow_disk_size_gb: int = 50, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + stage_2_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + enable_probabilistic_inference: bool = False, + quantiles: Optional[List[float]] = None, + encryption_spec_key_name: Optional[str] = None, + model_display_name: Optional[str] = None, + model_description: Optional[str] = None, + run_evaluation: bool = True, + group_columns: Optional[List[str]] = None, + group_total_weight: float = 0.0, + temporal_total_weight: float = 0.0, + group_temporal_total_weight: float = 0.0, + fields_to_exclude: FrozenSet[str] = frozenset(), +) -> Dict[str, Any]: + """Formats a set of parameters common across Vertex forecasting pipelines.""" + if not study_spec_parameters_override: + study_spec_parameters_override = [] + if not stage_1_tuner_worker_pool_specs_override: + stage_1_tuner_worker_pool_specs_override = [] + if not stage_2_trainer_worker_pool_specs_override: + stage_2_trainer_worker_pool_specs_override = [] + + if time_series_identifier_column: + logging.warning( + 'Deprecation warning: `time_series_identifier_column` will soon be' + ' deprecated in favor of `time_series_identifier_columns`. Please' + ' migrate workloads to use the new field.' + ) + time_series_identifier_columns = [time_series_identifier_column] + + parameter_values = {} + parameters = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'dataflow_service_account': dataflow_service_account, + 'evaluated_examples_bigquery_path': evaluated_examples_bigquery_path, + 'target_column': target_column, + 'optimization_objective': optimization_objective, + 'transformations': transformations, + 'train_budget_milli_node_hours': train_budget_milli_node_hours, + 'time_column': time_column, + 'time_series_identifier_columns': time_series_identifier_columns, + 'time_series_attribute_columns': time_series_attribute_columns, + 'available_at_forecast_columns': available_at_forecast_columns, + 'unavailable_at_forecast_columns': unavailable_at_forecast_columns, + 'forecast_horizon': forecast_horizon, + 'context_window': context_window, + 'window_predefined_column': window_predefined_column, + 'window_stride_length': window_stride_length, + 'window_max_count': window_max_count, + 'holiday_regions': holiday_regions, + 'stage_1_num_parallel_trials': stage_1_num_parallel_trials, + 'stage_1_tuning_result_artifact_uri': stage_1_tuning_result_artifact_uri, + 'stage_2_num_parallel_trials': stage_2_num_parallel_trials, + 'num_selected_trials': num_selected_trials, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'predefined_split_key': predefined_split_key, + 'timestamp_split_key': timestamp_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'weight_column': weight_column, + 'dataflow_subnetwork': dataflow_subnetwork, + 'feature_transform_engine_dataflow_machine_type': ( + feature_transform_engine_dataflow_machine_type + ), + 'feature_transform_engine_dataflow_max_num_workers': ( + feature_transform_engine_dataflow_max_num_workers + ), + 'feature_transform_engine_dataflow_disk_size_gb': ( + feature_transform_engine_dataflow_disk_size_gb + ), + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'feature_transform_engine_bigquery_staging_full_dataset_id': ( + feature_transform_engine_bigquery_staging_full_dataset_id + ), + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'study_spec_parameters_override': study_spec_parameters_override, + 'stage_1_tuner_worker_pool_specs_override': ( + stage_1_tuner_worker_pool_specs_override + ), + 'stage_2_trainer_worker_pool_specs_override': ( + stage_2_trainer_worker_pool_specs_override + ), + 'quantiles': quantiles, + 'encryption_spec_key_name': encryption_spec_key_name, + 'enable_probabilistic_inference': enable_probabilistic_inference, + 'model_display_name': model_display_name, + 'model_description': model_description, + 'run_evaluation': run_evaluation, + 'group_columns': group_columns, + 'group_total_weight': group_total_weight, + 'temporal_total_weight': temporal_total_weight, + 'group_temporal_total_weight': group_temporal_total_weight, + } + + # Filter out empty values and those excluded from the particular pipeline. + # (example: TFT and Seq2Seq don't support `quantiles`.) + parameter_values.update( + { + param: value + for param, value in parameters.items() + if value is not None and param not in fields_to_exclude + } + ) + return parameter_values + + +def get_learn_to_learn_forecasting_pipeline_and_parameters( + *, + project: str, + location: str, + root_dir: str, + target_column: str, + optimization_objective: str, + transformations: Dict[str, List[str]], + train_budget_milli_node_hours: float, + time_column: str, + time_series_identifier_columns: List[str], + time_series_identifier_column: Optional[str] = None, + time_series_attribute_columns: Optional[List[str]] = None, + available_at_forecast_columns: Optional[List[str]] = None, + unavailable_at_forecast_columns: Optional[List[str]] = None, + forecast_horizon: Optional[int] = None, + context_window: Optional[int] = None, + evaluated_examples_bigquery_path: Optional[str] = None, + window_predefined_column: Optional[str] = None, + window_stride_length: Optional[int] = None, + window_max_count: Optional[int] = None, + holiday_regions: Optional[List[str]] = None, + stage_1_num_parallel_trials: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + stage_2_num_parallel_trials: Optional[int] = None, + num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + dataflow_service_account: Optional[str] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + feature_transform_engine_bigquery_staging_full_dataset_id: str = '', + feature_transform_engine_dataflow_machine_type: str = 'n1-standard-16', + feature_transform_engine_dataflow_max_num_workers: int = 10, + feature_transform_engine_dataflow_disk_size_gb: int = 40, + evaluation_batch_predict_machine_type: str = 'n1-standard-16', + evaluation_batch_predict_starting_replica_count: int = 25, + evaluation_batch_predict_max_replica_count: int = 25, + evaluation_dataflow_machine_type: str = 'n1-standard-16', + evaluation_dataflow_max_num_workers: int = 25, + evaluation_dataflow_disk_size_gb: int = 50, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + stage_2_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + enable_probabilistic_inference: bool = False, + quantiles: Optional[List[float]] = None, + encryption_spec_key_name: Optional[str] = None, + model_display_name: Optional[str] = None, + model_description: Optional[str] = None, + run_evaluation: bool = True, + group_columns: Optional[List[str]] = None, + group_total_weight: float = 0.0, + temporal_total_weight: float = 0.0, + group_temporal_total_weight: float = 0.0, +) -> Tuple[str, Dict[str, Any]]: + """Returns l2l_forecasting pipeline and formatted parameters. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + optimization_objective: "minimize-rmse", "minimize-mae", "minimize-rmsle", + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + "minimize-quantile-loss". + transformations: Dict mapping auto and/or type-resolutions to feature + columns. The supported types are: auto, categorical, numeric, text, and + timestamp. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + time_column: The column that indicates the time. + time_series_identifier_columns: The columns which distinguish different time + series. + time_series_identifier_column: [Deprecated] The column which distinguishes + different time series. + time_series_attribute_columns: The columns that are invariant across the + same time series. + available_at_forecast_columns: The columns that are available at the + forecast time. + unavailable_at_forecast_columns: The columns that are unavailable at the + forecast time. + forecast_horizon: The length of the horizon. + context_window: The length of the context window. + evaluated_examples_bigquery_path: The bigquery dataset to write the + predicted examples into for evaluation, in the format + `bq://project.dataset`. + window_predefined_column: The column that indicate the start of each window. + window_stride_length: The stride length to generate the window. + window_max_count: The maximum number of windows that will be generated. + holiday_regions: The geographical regions where the holiday effect is + applied in modeling. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + num_selected_trials: Number of selected trails. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + predefined_split_key: The predefined_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: The test fraction. + weight_column: The weight column name. + dataflow_service_account: The full service account name. + dataflow_subnetwork: The dataflow subnetwork. + dataflow_use_public_ips: `True` to enable dataflow public IPs. + feature_transform_engine_bigquery_staging_full_dataset_id: The full id of + the feature transform engine staging dataset. + feature_transform_engine_dataflow_machine_type: The dataflow machine type of + the feature transform engine. + feature_transform_engine_dataflow_max_num_workers: The max number of + dataflow workers of the feature transform engine. + feature_transform_engine_dataflow_disk_size_gb: The disk size of the + dataflow workers of the feature transform engine. + evaluation_batch_predict_machine_type: Machine type for the batch prediction + job in evaluation, such as 'n1-standard-16'. + evaluation_batch_predict_starting_replica_count: Number of replicas to use + in the batch prediction cluster at startup time. + evaluation_batch_predict_max_replica_count: The maximum count of replicas + the batch prediction job can scale to. + evaluation_dataflow_machine_type: Machine type for the dataflow job in + evaluation, such as 'n1-standard-16'. + evaluation_dataflow_max_num_workers: Maximum number of dataflow workers. + evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow. + study_spec_parameters_override: The list for overriding study spec. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding + stage 1 tuner worker pool spec. + stage_2_trainer_worker_pool_specs_override: The dictionary for overriding + stage 2 trainer worker pool spec. + enable_probabilistic_inference: If probabilistic inference is enabled, the + model will fit a distribution that captures the uncertainty of a + prediction. If quantiles are specified, then the quantiles of the + distribution are also returned. + quantiles: Quantiles to use for probabilistic inference. Up to 5 quantiles + are allowed of values between 0 and 1, exclusive. Represents the quantiles + to use for that objective. Quantiles must be unique. + encryption_spec_key_name: The KMS key name. + model_display_name: Optional display name for model. + model_description: Optional description. + run_evaluation: `True` to evaluate the ensembled model on the test split. + group_columns: A list of time series attribute column names that define the + time series hierarchy. + group_total_weight: The weight of the loss for predictions aggregated over + time series in the same group. + temporal_total_weight: The weight of the loss for predictions aggregated + over the horizon for a single time series. + group_temporal_total_weight: The weight of the loss for predictions + aggregated over both the horizon and time series in the same hierarchy + group. + """ + parameter_values = _get_base_forecasting_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + evaluated_examples_bigquery_path=evaluated_examples_bigquery_path, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + time_column=time_column, + dataflow_service_account=dataflow_service_account, + time_series_identifier_columns=time_series_identifier_columns, + time_series_identifier_column=time_series_identifier_column, + time_series_attribute_columns=time_series_attribute_columns, + available_at_forecast_columns=available_at_forecast_columns, + unavailable_at_forecast_columns=unavailable_at_forecast_columns, + forecast_horizon=forecast_horizon, + context_window=context_window, + window_predefined_column=window_predefined_column, + window_stride_length=window_stride_length, + window_max_count=window_max_count, + holiday_regions=holiday_regions, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + num_selected_trials=num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + dataflow_use_public_ips=dataflow_use_public_ips, + dataflow_subnetwork=dataflow_subnetwork, + feature_transform_engine_bigquery_staging_full_dataset_id=feature_transform_engine_bigquery_staging_full_dataset_id, + feature_transform_engine_dataflow_machine_type=feature_transform_engine_dataflow_machine_type, + feature_transform_engine_dataflow_max_num_workers=feature_transform_engine_dataflow_max_num_workers, + feature_transform_engine_dataflow_disk_size_gb=feature_transform_engine_dataflow_disk_size_gb, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + study_spec_parameters_override=study_spec_parameters_override, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + stage_2_trainer_worker_pool_specs_override=stage_2_trainer_worker_pool_specs_override, + quantiles=quantiles, + encryption_spec_key_name=encryption_spec_key_name, + enable_probabilistic_inference=enable_probabilistic_inference, + model_display_name=model_display_name, + model_description=model_description, + run_evaluation=run_evaluation, + group_columns=group_columns, + group_total_weight=group_total_weight, + temporal_total_weight=temporal_total_weight, + group_temporal_total_weight=group_temporal_total_weight, + ) + + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, + 'learn_to_learn_forecasting_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_time_series_dense_encoder_forecasting_pipeline_and_parameters( + *, + project: str, + location: str, + root_dir: str, + target_column: str, + optimization_objective: str, + transformations: Dict[str, List[str]], + train_budget_milli_node_hours: float, + time_column: str, + time_series_identifier_columns: List[str], + time_series_identifier_column: Optional[str] = None, + time_series_attribute_columns: Optional[List[str]] = None, + available_at_forecast_columns: Optional[List[str]] = None, + unavailable_at_forecast_columns: Optional[List[str]] = None, + forecast_horizon: Optional[int] = None, + context_window: Optional[int] = None, + evaluated_examples_bigquery_path: Optional[str] = None, + window_predefined_column: Optional[str] = None, + window_stride_length: Optional[int] = None, + window_max_count: Optional[int] = None, + holiday_regions: Optional[List[str]] = None, + stage_1_num_parallel_trials: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + stage_2_num_parallel_trials: Optional[int] = None, + num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + dataflow_service_account: Optional[str] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + feature_transform_engine_bigquery_staging_full_dataset_id: str = '', + feature_transform_engine_dataflow_machine_type: str = 'n1-standard-16', + feature_transform_engine_dataflow_max_num_workers: int = 10, + feature_transform_engine_dataflow_disk_size_gb: int = 40, + evaluation_batch_predict_machine_type: str = 'n1-standard-16', + evaluation_batch_predict_starting_replica_count: int = 25, + evaluation_batch_predict_max_replica_count: int = 25, + evaluation_dataflow_machine_type: str = 'n1-standard-16', + evaluation_dataflow_max_num_workers: int = 25, + evaluation_dataflow_disk_size_gb: int = 50, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + stage_2_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + enable_probabilistic_inference: bool = False, + quantiles: Optional[List[float]] = None, + encryption_spec_key_name: Optional[str] = None, + model_display_name: Optional[str] = None, + model_description: Optional[str] = None, + run_evaluation: bool = True, + group_columns: Optional[List[str]] = None, + group_total_weight: float = 0.0, + temporal_total_weight: float = 0.0, + group_temporal_total_weight: float = 0.0, +) -> Tuple[str, Dict[str, Any]]: + """Returns timeseries_dense_encoder_forecasting pipeline and parameters. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + optimization_objective: "minimize-rmse", "minimize-mae", "minimize-rmsle", + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + "minimize-quantile-loss". + transformations: Dict mapping auto and/or type-resolutions to feature + columns. The supported types are: auto, categorical, numeric, text, and + timestamp. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + time_column: The column that indicates the time. + time_series_identifier_columns: The columns which distinguish different time + series. + time_series_identifier_column: [Deprecated] The column which distinguishes + different time series. + time_series_attribute_columns: The columns that are invariant across the + same time series. + available_at_forecast_columns: The columns that are available at the + forecast time. + unavailable_at_forecast_columns: The columns that are unavailable at the + forecast time. + forecast_horizon: The length of the horizon. + context_window: The length of the context window. + evaluated_examples_bigquery_path: The bigquery dataset to write the + predicted examples into for evaluation, in the format + `bq://project.dataset`. + window_predefined_column: The column that indicate the start of each window. + window_stride_length: The stride length to generate the window. + window_max_count: The maximum number of windows that will be generated. + holiday_regions: The geographical regions where the holiday effect is + applied in modeling. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + num_selected_trials: Number of selected trails. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + predefined_split_key: The predefined_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: The test fraction. + weight_column: The weight column name. + dataflow_service_account: The full service account name. + dataflow_subnetwork: The dataflow subnetwork. + dataflow_use_public_ips: `True` to enable dataflow public IPs. + feature_transform_engine_bigquery_staging_full_dataset_id: The full id of + the feature transform engine staging dataset. + feature_transform_engine_dataflow_machine_type: The dataflow machine type of + the feature transform engine. + feature_transform_engine_dataflow_max_num_workers: The max number of + dataflow workers of the feature transform engine. + feature_transform_engine_dataflow_disk_size_gb: The disk size of the + dataflow workers of the feature transform engine. + evaluation_batch_predict_machine_type: Machine type for the batch prediction + job in evaluation, such as 'n1-standard-16'. + evaluation_batch_predict_starting_replica_count: Number of replicas to use + in the batch prediction cluster at startup time. + evaluation_batch_predict_max_replica_count: The maximum count of replicas + the batch prediction job can scale to. + evaluation_dataflow_machine_type: Machine type for the dataflow job in + evaluation, such as 'n1-standard-16'. + evaluation_dataflow_max_num_workers: Maximum number of dataflow workers. + evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow. + study_spec_parameters_override: The list for overriding study spec. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding + stage 1 tuner worker pool spec. + stage_2_trainer_worker_pool_specs_override: The dictionary for overriding + stage 2 trainer worker pool spec. + enable_probabilistic_inference: If probabilistic inference is enabled, the + model will fit a distribution that captures the uncertainty of a + prediction. If quantiles are specified, then the quantiles of the + distribution are also returned. + quantiles: Quantiles to use for probabilistic inference. Up to 5 quantiles + are allowed of values between 0 and 1, exclusive. Represents the quantiles + to use for that objective. Quantiles must be unique. + encryption_spec_key_name: The KMS key name. + model_display_name: Optional display name for model. + model_description: Optional description. + run_evaluation: `True` to evaluate the ensembled model on the test split. + group_columns: A list of time series attribute column names that define the + time series hierarchy. + group_total_weight: The weight of the loss for predictions aggregated over + time series in the same group. + temporal_total_weight: The weight of the loss for predictions aggregated + over the horizon for a single time series. + group_temporal_total_weight: The weight of the loss for predictions + aggregated over both the horizon and time series in the same hierarchy + group. + """ + + parameter_values = _get_base_forecasting_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + evaluated_examples_bigquery_path=evaluated_examples_bigquery_path, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + time_column=time_column, + dataflow_service_account=dataflow_service_account, + time_series_identifier_columns=time_series_identifier_columns, + time_series_identifier_column=time_series_identifier_column, + time_series_attribute_columns=time_series_attribute_columns, + available_at_forecast_columns=available_at_forecast_columns, + unavailable_at_forecast_columns=unavailable_at_forecast_columns, + forecast_horizon=forecast_horizon, + context_window=context_window, + window_predefined_column=window_predefined_column, + window_stride_length=window_stride_length, + window_max_count=window_max_count, + holiday_regions=holiday_regions, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + num_selected_trials=num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + dataflow_use_public_ips=dataflow_use_public_ips, + dataflow_subnetwork=dataflow_subnetwork, + feature_transform_engine_bigquery_staging_full_dataset_id=feature_transform_engine_bigquery_staging_full_dataset_id, + feature_transform_engine_dataflow_machine_type=feature_transform_engine_dataflow_machine_type, + feature_transform_engine_dataflow_max_num_workers=feature_transform_engine_dataflow_max_num_workers, + feature_transform_engine_dataflow_disk_size_gb=feature_transform_engine_dataflow_disk_size_gb, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + study_spec_parameters_override=study_spec_parameters_override, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + stage_2_trainer_worker_pool_specs_override=stage_2_trainer_worker_pool_specs_override, + quantiles=quantiles, + encryption_spec_key_name=encryption_spec_key_name, + enable_probabilistic_inference=enable_probabilistic_inference, + model_display_name=model_display_name, + model_description=model_description, + run_evaluation=run_evaluation, + group_columns=group_columns, + group_total_weight=group_total_weight, + temporal_total_weight=temporal_total_weight, + group_temporal_total_weight=group_temporal_total_weight, + ) + + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, + 'time_series_dense_encoder_forecasting_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_temporal_fusion_transformer_forecasting_pipeline_and_parameters( + *, + project: str, + location: str, + root_dir: str, + target_column: str, + optimization_objective: str, + transformations: Dict[str, List[str]], + train_budget_milli_node_hours: float, + time_column: str, + time_series_identifier_columns: List[str], + time_series_identifier_column: Optional[str] = None, + time_series_attribute_columns: Optional[List[str]] = None, + available_at_forecast_columns: Optional[List[str]] = None, + unavailable_at_forecast_columns: Optional[List[str]] = None, + forecast_horizon: Optional[int] = None, + context_window: Optional[int] = None, + evaluated_examples_bigquery_path: Optional[str] = None, + window_predefined_column: Optional[str] = None, + window_stride_length: Optional[int] = None, + window_max_count: Optional[int] = None, + holiday_regions: Optional[List[str]] = None, + stage_1_num_parallel_trials: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + stage_2_num_parallel_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + dataflow_service_account: Optional[str] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + feature_transform_engine_bigquery_staging_full_dataset_id: str = '', + feature_transform_engine_dataflow_machine_type: str = 'n1-standard-16', + feature_transform_engine_dataflow_max_num_workers: int = 10, + feature_transform_engine_dataflow_disk_size_gb: int = 40, + evaluation_batch_predict_machine_type: str = 'n1-standard-16', + evaluation_batch_predict_starting_replica_count: int = 25, + evaluation_batch_predict_max_replica_count: int = 25, + evaluation_dataflow_machine_type: str = 'n1-standard-16', + evaluation_dataflow_max_num_workers: int = 25, + evaluation_dataflow_disk_size_gb: int = 50, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + stage_2_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + encryption_spec_key_name: Optional[str] = None, + model_display_name: Optional[str] = None, + model_description: Optional[str] = None, + run_evaluation: bool = True, +): + """Returns tft_forecasting pipeline and formatted parameters. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + optimization_objective: "minimize-rmse", "minimize-mae", "minimize-rmsle", + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + "minimize-quantile-loss". + transformations: Dict mapping auto and/or type-resolutions to feature + columns. The supported types are: auto, categorical, numeric, text, and + timestamp. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + time_column: The column that indicates the time. + time_series_identifier_columns: The columns which distinguish different time + series. + time_series_identifier_column: [Deprecated] The column which distinguishes + different time series. + time_series_attribute_columns: The columns that are invariant across the + same time series. + available_at_forecast_columns: The columns that are available at the + forecast time. + unavailable_at_forecast_columns: The columns that are unavailable at the + forecast time. + forecast_horizon: The length of the horizon. + context_window: The length of the context window. + evaluated_examples_bigquery_path: The bigquery dataset to write the + predicted examples into for evaluation, in the format + `bq://project.dataset`. + window_predefined_column: The column that indicate the start of each window. + window_stride_length: The stride length to generate the window. + window_max_count: The maximum number of windows that will be generated. + holiday_regions: The geographical regions where the holiday effect is + applied in modeling. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + predefined_split_key: The predefined_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: The test fraction. + weight_column: The weight column name. + dataflow_service_account: The full service account name. + dataflow_subnetwork: The dataflow subnetwork. + dataflow_use_public_ips: `True` to enable dataflow public IPs. + feature_transform_engine_bigquery_staging_full_dataset_id: The full id of + the feature transform engine staging dataset. + feature_transform_engine_dataflow_machine_type: The dataflow machine type of + the feature transform engine. + feature_transform_engine_dataflow_max_num_workers: The max number of + dataflow workers of the feature transform engine. + feature_transform_engine_dataflow_disk_size_gb: The disk size of the + dataflow workers of the feature transform engine. + evaluation_batch_predict_machine_type: Machine type for the batch prediction + job in evaluation, such as 'n1-standard-16'. + evaluation_batch_predict_starting_replica_count: Number of replicas to use + in the batch prediction cluster at startup time. + evaluation_batch_predict_max_replica_count: The maximum count of replicas + the batch prediction job can scale to. + evaluation_dataflow_machine_type: Machine type for the dataflow job in + evaluation, such as 'n1-standard-16'. + evaluation_dataflow_max_num_workers: Maximum number of dataflow workers. + evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow. + study_spec_parameters_override: The list for overriding study spec. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding + stage 1 tuner worker pool spec. + stage_2_trainer_worker_pool_specs_override: The dictionary for overriding + stage 2 trainer worker pool spec. + encryption_spec_key_name: The KMS key name. + model_display_name: Optional display name for model. + model_description: Optional description. + run_evaluation: `True` to evaluate the ensembled model on the test split. + """ + # TFT should only have 1 selected trial to freeze the ensemble size at 1. + excluded_parameters = _RETAIL_MODEL_DISABLED_OPTIONS.union({ + 'num_selected_trials', + }) + parameter_values = _get_base_forecasting_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + evaluated_examples_bigquery_path=evaluated_examples_bigquery_path, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + time_column=time_column, + dataflow_service_account=dataflow_service_account, + time_series_identifier_columns=time_series_identifier_columns, + time_series_identifier_column=time_series_identifier_column, + time_series_attribute_columns=time_series_attribute_columns, + available_at_forecast_columns=available_at_forecast_columns, + unavailable_at_forecast_columns=unavailable_at_forecast_columns, + forecast_horizon=forecast_horizon, + context_window=context_window, + window_predefined_column=window_predefined_column, + window_stride_length=window_stride_length, + window_max_count=window_max_count, + holiday_regions=holiday_regions, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + dataflow_use_public_ips=dataflow_use_public_ips, + dataflow_subnetwork=dataflow_subnetwork, + feature_transform_engine_bigquery_staging_full_dataset_id=feature_transform_engine_bigquery_staging_full_dataset_id, + feature_transform_engine_dataflow_machine_type=feature_transform_engine_dataflow_machine_type, + feature_transform_engine_dataflow_max_num_workers=feature_transform_engine_dataflow_max_num_workers, + feature_transform_engine_dataflow_disk_size_gb=feature_transform_engine_dataflow_disk_size_gb, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + study_spec_parameters_override=study_spec_parameters_override, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + stage_2_trainer_worker_pool_specs_override=stage_2_trainer_worker_pool_specs_override, + encryption_spec_key_name=encryption_spec_key_name, + model_display_name=model_display_name, + model_description=model_description, + run_evaluation=run_evaluation, + fields_to_exclude=excluded_parameters, + ) + + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, + 'temporal_fusion_transformer_forecasting_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_sequence_to_sequence_forecasting_pipeline_and_parameters( + *, + project: str, + location: str, + root_dir: str, + target_column: str, + optimization_objective: str, + transformations: Dict[str, List[str]], + train_budget_milli_node_hours: float, + time_column: str, + time_series_identifier_columns: List[str], + time_series_identifier_column: Optional[str] = None, + time_series_attribute_columns: Optional[List[str]] = None, + available_at_forecast_columns: Optional[List[str]] = None, + unavailable_at_forecast_columns: Optional[List[str]] = None, + forecast_horizon: Optional[int] = None, + context_window: Optional[int] = None, + evaluated_examples_bigquery_path: Optional[str] = None, + window_predefined_column: Optional[str] = None, + window_stride_length: Optional[int] = None, + window_max_count: Optional[int] = None, + holiday_regions: Optional[List[str]] = None, + stage_1_num_parallel_trials: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + stage_2_num_parallel_trials: Optional[int] = None, + num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + dataflow_service_account: Optional[str] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + feature_transform_engine_bigquery_staging_full_dataset_id: str = '', + feature_transform_engine_dataflow_machine_type: str = 'n1-standard-16', + feature_transform_engine_dataflow_max_num_workers: int = 10, + feature_transform_engine_dataflow_disk_size_gb: int = 40, + evaluation_batch_predict_machine_type: str = 'n1-standard-16', + evaluation_batch_predict_starting_replica_count: int = 25, + evaluation_batch_predict_max_replica_count: int = 25, + evaluation_dataflow_machine_type: str = 'n1-standard-16', + evaluation_dataflow_max_num_workers: int = 25, + evaluation_dataflow_disk_size_gb: int = 50, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + stage_2_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + encryption_spec_key_name: Optional[str] = None, + model_display_name: Optional[str] = None, + model_description: Optional[str] = None, + run_evaluation: bool = True, +): + """Returns seq2seq forecasting pipeline and formatted parameters. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + optimization_objective: "minimize-rmse", "minimize-mae", "minimize-rmsle", + "minimize-rmspe", "minimize-wape-mae", "minimize-mape", or + "minimize-quantile-loss". + transformations: Dict mapping auto and/or type-resolutions to feature + columns. The supported types are: auto, categorical, numeric, text, and + timestamp. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + time_column: The column that indicates the time. + time_series_identifier_columns: The columns which distinguish different time + series. + time_series_identifier_column: [Deprecated] The column which distinguishes + different time series. + time_series_attribute_columns: The columns that are invariant across the + same time series. + available_at_forecast_columns: The columns that are available at the + forecast time. + unavailable_at_forecast_columns: The columns that are unavailable at the + forecast time. + forecast_horizon: The length of the horizon. + context_window: The length of the context window. + evaluated_examples_bigquery_path: The bigquery dataset to write the + predicted examples into for evaluation, in the format + `bq://project.dataset`. + window_predefined_column: The column that indicate the start of each window. + window_stride_length: The stride length to generate the window. + window_max_count: The maximum number of windows that will be generated. + holiday_regions: The geographical regions where the holiday effect is + applied in modeling. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + num_selected_trials: Number of selected trails. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + predefined_split_key: The predefined_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: The test fraction. + weight_column: The weight column name. + dataflow_service_account: The full service account name. + dataflow_subnetwork: The dataflow subnetwork. + dataflow_use_public_ips: `True` to enable dataflow public IPs. + feature_transform_engine_bigquery_staging_full_dataset_id: The full id of + the feature transform engine staging dataset. + feature_transform_engine_dataflow_machine_type: The dataflow machine type of + the feature transform engine. + feature_transform_engine_dataflow_max_num_workers: The max number of + dataflow workers of the feature transform engine. + feature_transform_engine_dataflow_disk_size_gb: The disk size of the + dataflow workers of the feature transform engine. + evaluation_batch_predict_machine_type: Machine type for the batch prediction + job in evaluation, such as 'n1-standard-16'. + evaluation_batch_predict_starting_replica_count: Number of replicas to use + in the batch prediction cluster at startup time. + evaluation_batch_predict_max_replica_count: The maximum count of replicas + the batch prediction job can scale to. + evaluation_dataflow_machine_type: Machine type for the dataflow job in + evaluation, such as 'n1-standard-16'. + evaluation_dataflow_max_num_workers: Maximum number of dataflow workers. + evaluation_dataflow_disk_size_gb: The disk space in GB for dataflow. + study_spec_parameters_override: The list for overriding study spec. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding + stage 1 tuner worker pool spec. + stage_2_trainer_worker_pool_specs_override: The dictionary for overriding + stage 2 trainer worker pool spec. + encryption_spec_key_name: The KMS key name. + model_display_name: Optional display name for model. + model_description: Optional description. + run_evaluation: `True` to evaluate the ensembled model on the test split. + """ + + parameter_values = _get_base_forecasting_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + evaluated_examples_bigquery_path=evaluated_examples_bigquery_path, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + time_column=time_column, + dataflow_service_account=dataflow_service_account, + time_series_identifier_columns=time_series_identifier_columns, + time_series_identifier_column=time_series_identifier_column, + time_series_attribute_columns=time_series_attribute_columns, + available_at_forecast_columns=available_at_forecast_columns, + unavailable_at_forecast_columns=unavailable_at_forecast_columns, + forecast_horizon=forecast_horizon, + context_window=context_window, + window_predefined_column=window_predefined_column, + window_stride_length=window_stride_length, + window_max_count=window_max_count, + holiday_regions=holiday_regions, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + num_selected_trials=num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + dataflow_use_public_ips=dataflow_use_public_ips, + dataflow_subnetwork=dataflow_subnetwork, + feature_transform_engine_bigquery_staging_full_dataset_id=feature_transform_engine_bigquery_staging_full_dataset_id, + feature_transform_engine_dataflow_machine_type=feature_transform_engine_dataflow_machine_type, + feature_transform_engine_dataflow_max_num_workers=feature_transform_engine_dataflow_max_num_workers, + feature_transform_engine_dataflow_disk_size_gb=feature_transform_engine_dataflow_disk_size_gb, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + study_spec_parameters_override=study_spec_parameters_override, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + stage_2_trainer_worker_pool_specs_override=stage_2_trainer_worker_pool_specs_override, + encryption_spec_key_name=encryption_spec_key_name, + model_display_name=model_display_name, + model_description=model_description, + run_evaluation=run_evaluation, + fields_to_exclude=_RETAIL_MODEL_DISABLED_OPTIONS, + ) + + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, + 'sequence_to_sequence_forecasting_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/__init__.py new file mode 100644 index 0000000000..4268da69ff --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/__init__.py @@ -0,0 +1,56 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Preview AutoML tabular components.""" + +import os + +from google_cloud_pipeline_components.preview.automl.tabular.auto_feature_engineering import automated_feature_engineering as AutoFeatureEngineeringOp +from google_cloud_pipeline_components.preview.automl.tabular.distillation_stage_feature_transform_engine import distillation_stage_feature_transform_engine as DistillationStageFeatureTransformEngineOp +from google_cloud_pipeline_components.preview.automl.tabular.feature_selection import tabular_feature_ranking_and_selection as FeatureSelectionOp +from google_cloud_pipeline_components.preview.automl.tabular.feature_transform_engine import feature_transform_engine as FeatureTransformEngineOp +from google_cloud_pipeline_components.preview.automl.tabular.tabnet_hyperparameter_tuning_job import tabnet_hyperparameter_tuning_job as TabNetHyperparameterTuningJobOp +from google_cloud_pipeline_components.preview.automl.tabular.tabnet_trainer import tabnet_trainer as TabNetTrainerOp +from google_cloud_pipeline_components.preview.automl.tabular.wide_and_deep_hyperparameter_tuning_job import wide_and_deep_hyperparameter_tuning_job as WideAndDeepHyperparameterTuningJobOp +from google_cloud_pipeline_components.preview.automl.tabular.wide_and_deep_trainer import wide_and_deep_trainer as WideAndDeepTrainerOp +from google_cloud_pipeline_components.preview.automl.tabular.xgboost_hyperparameter_tuning_job import xgboost_hyperparameter_tuning_job as XGBoostHyperparameterTuningJobOp +from google_cloud_pipeline_components.preview.automl.tabular.xgboost_trainer import xgboost_trainer as XGBoostTrainerOp +from kfp import components + +__all__ = [ + 'AutoFeatureEngineeringOp', + 'FeatureSelectionOp', + 'WideAndDeepHyperparameterTuningJobOp', + 'WideAndDeepTrainerOp', + 'TabNetHyperparameterTuningJobOp', + 'TabNetTrainerOp', + 'FeatureTransformEngineOp', + 'DistillationStageFeatureTransformEngineOp', + 'XGBoostHyperparameterTuningJobOp', + 'XGBoostTrainerOp', +] + +tabnet_trainer_pipeline = components.load_component_from_file( + # Note, please don't name it as `component.yaml` which will conflict with + # the generated file. + os.path.join(os.path.dirname(__file__), 'tabnet_trainer_pipeline.yaml') +) + +wide_and_deep_trainer_pipeline = components.load_component_from_file( + # Note, please don't name it as `component.yaml` which will conflict with + # the generated file. + os.path.join( + os.path.dirname(__file__), 'wide_and_deep_trainer_pipeline.yaml' + ) +) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py new file mode 100644 index 0000000000..b6d87ac94f --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/auto_feature_engineering.py @@ -0,0 +1,91 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Auto Feature Engineering component spec.""" + +from typing import Optional + +from kfp import dsl + + +@dsl.container_component +def automated_feature_engineering( + root_dir: str, + project: str, + location: str, + gcp_resources: dsl.OutputPath(str), + materialized_data: dsl.Output[dsl.Dataset], + feature_ranking: dsl.Output[dsl.Artifact], + target_column: Optional[str] = '', + weight_column: Optional[str] = '', + data_source_csv_filenames: Optional[str] = '', + data_source_bigquery_table_path: Optional[str] = '', + bigquery_staging_full_dataset_id: Optional[str] = '', + materialized_examples_format: Optional[str] = 'tfrecords_gzip', +): + """Find the top features from the dataset.""" + # fmt: off + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "auto-feature-engineering-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ), + ( + '"job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-16"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "args": ["feature_engineering", "--project=', project, + '", "--location=', location, '", "--data_source_bigquery_table_path=', + data_source_bigquery_table_path, + '", "--target_column=', + target_column, + '", "--weight_column=', + weight_column, + '", "--bigquery_staging_full_dataset_id=', + bigquery_staging_full_dataset_id, + '", "--materialized_data_path=', + root_dir, f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/materialized_data", ', + ' "--materialized_examples_path=', + root_dir, f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/materialized", ' + ' "--error_file_path=', + root_dir, f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb", ' + ' "--materialized_data_artifact_path=', + materialized_data.uri, + '", "--feature_ranking_path=', + feature_ranking.uri, '"]}}]}}' + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml new file mode 100644 index 0000000000..ed45c6e92c --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_feature_selection_pipeline.yaml @@ -0,0 +1,11488 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-feature-selection-pipeline +# Description: The AutoML Tabular pipeline. +# Inputs: +# additional_experiments: dict +# apply_feature_selection_tuning: bool [Default: False] +# cv_trainer_worker_pool_specs_override: list +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# disable_early_stopping: bool [Default: False] +# distill_batch_predict_machine_type: str [Default: 'n1-standard-16'] +# distill_batch_predict_max_replica_count: int [Default: 25.0] +# distill_batch_predict_starting_replica_count: int [Default: 25.0] +# enable_probabilistic_inference: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# evaluation_batch_explain_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_explain_max_replica_count: int [Default: 10.0] +# evaluation_batch_explain_starting_replica_count: int [Default: 10.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# export_additional_model_without_custom_ops: bool [Default: False] +# fast_testing: bool [Default: False] +# location: str +# max_selected_features: int [Default: 1000.0] +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# optimization_objective: str +# optimization_objective_precision_value: float [Default: -1.0] +# optimization_objective_recall_value: float [Default: -1.0] +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# quantiles: list +# root_dir: str +# run_distillation: bool [Default: False] +# run_evaluation: bool [Default: False] +# stage_1_num_parallel_trials: int [Default: 35.0] +# stage_1_tuner_worker_pool_specs_override: list +# stage_2_num_parallel_trials: int [Default: 35.0] +# stage_2_num_selected_trials: int [Default: 5.0] +# stats_and_example_gen_dataflow_disk_size_gb: int [Default: 40.0] +# stats_and_example_gen_dataflow_machine_type: str [Default: 'n1-standard-16'] +# stats_and_example_gen_dataflow_max_num_workers: int [Default: 25.0] +# stratified_split_key: str [Default: ''] +# study_spec_parameters_override: list +# target_column: str +# test_fraction: float [Default: -1.0] +# timestamp_split_key: str [Default: ''] +# train_budget_milli_node_hours: float +# training_fraction: float [Default: -1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# transformations: str +# validation_fraction: float [Default: -1.0] +# weight_column: str [Default: ''] +# Outputs: +# feature-attribution-2-feature_attributions: system.Metrics +# feature-attribution-3-feature_attributions: system.Metrics +# feature-attribution-feature_attributions: system.Metrics +# model-evaluation-2-evaluation_metrics: system.Metrics +# model-evaluation-3-evaluation_metrics: system.Metrics +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-cv-trainer: + executorLabel: exec-automl-tabular-cv-trainer + inputDefinitions: + artifacts: + materialized_cv_splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized cross-validation splits. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: AutoML Tabular tuning result. + parameters: + deadline_hours: + description: Number of hours the cross-validation trainer should run. + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-cv-trainer-2: + executorLabel: exec-automl-tabular-cv-trainer-2 + inputDefinitions: + artifacts: + materialized_cv_splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized cross-validation splits. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: AutoML Tabular tuning result. + parameters: + deadline_hours: + description: Number of hours the cross-validation trainer should run. + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble: + executorLabel: exec-automl-tabular-ensemble + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble-2: + executorLabel: exec-automl-tabular-ensemble-2 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble-3: + executorLabel: exec-automl-tabular-ensemble-3 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-infra-validator: + executorLabel: exec-automl-tabular-infra-validator + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-infra-validator-2: + executorLabel: exec-automl-tabular-infra-validator-2 + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-infra-validator-3: + executorLabel: exec-automl-tabular-infra-validator-3 + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-stage-1-tuner: + executorLabel: exec-automl-tabular-stage-1-tuner + inputDefinitions: + artifacts: + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + disable_early_stopping: + defaultValue: false + description: 'True if disable early stopping. Default + + value is false.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "model_type","categorical_value_spec": {"values": + + ["nn"]}}]' + isOptional: true + parameterType: LIST + tune_feature_selection_rate: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-stage-1-tuner-2: + executorLabel: exec-automl-tabular-stage-1-tuner-2 + inputDefinitions: + artifacts: + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + disable_early_stopping: + defaultValue: false + description: 'True if disable early stopping. Default + + value is false.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "model_type","categorical_value_spec": {"values": + + ["nn"]}}]' + isOptional: true + parameterType: LIST + tune_feature_selection_rate: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-transform: + executorLabel: exec-automl-tabular-transform + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The eval split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The test split. + train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The train split. + parameters: + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized test split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized train split. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-transform-2: + executorLabel: exec-automl-tabular-transform-2 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The eval split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The test split. + train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The train split. + parameters: + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized test split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized train split. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-bool-identity-2: + executorLabel: exec-bool-identity-2 + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-bool-identity-3: + executorLabel: exec-bool-identity-3 + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-calculate-training-parameters: + executorLabel: exec-calculate-training-parameters + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + run_distillation: + description: Whether to run distill in the training pipeline. + parameterType: BOOLEAN + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + reduce_search_space_mode: + parameterType: STRING + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_num_selected_trials: + parameterType: NUMBER_INTEGER + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-calculate-training-parameters-2: + executorLabel: exec-calculate-training-parameters-2 + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + run_distillation: + description: Whether to run distill in the training pipeline. + parameterType: BOOLEAN + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + reduce_search_space_mode: + parameterType: STRING + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_num_selected_trials: + parameterType: NUMBER_INTEGER + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-check-if-binary-classification: + executorLabel: exec-check-if-binary-classification + inputDefinitions: + artifacts: + example_gen_metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: metadata generated by example gen. + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-3 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-3 + tasks: + automl-tabular-cv-trainer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-cv-trainer + dependentTasks: + - calculate-training-parameters + - importer + inputs: + artifacts: + materialized_cv_splits: + componentInputArtifact: pipelinechannel--merge-materialized-splits-splits + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + taskInfo: + name: automl-tabular-cv-trainer + automl-tabular-ensemble: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble + dependentTasks: + - automl-tabular-cv-trainer + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-cv-trainer + warmup_data: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble + automl-tabular-infra-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator + dependentTasks: + - automl-tabular-ensemble + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + taskInfo: + name: automl-tabular-infra-validator + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + calculate-training-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: true + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters + condition-3: + componentRef: + name: comp-condition-3 + dependentTasks: + - automl-tabular-ensemble + - bool-identity + - model-upload + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble + pipelinechannel--automl-tabular-ensemble-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--automl-tabular-ensemble-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + runtimeValue: + constant: '' + taskInfo: + name: importer + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-tabular-ensemble + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-transform-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--merge-materialized-splits-splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--purge-unused-features-output_metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + feature-attribution: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution + dependentTasks: + - model-batch-explanation + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution + model-batch-explanation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - feature-attribution + - model-evaluation + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_paths: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + dataset_type: + runtimeValue: + constant: tf-record + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-4: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-5 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-7 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-5 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-7 + tasks: + automl-tabular-cv-trainer-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-cv-trainer-2 + dependentTasks: + - automl-tabular-stage-1-tuner + - calculate-training-parameters-2 + inputs: + artifacts: + materialized_cv_splits: + componentInputArtifact: pipelinechannel--merge-materialized-splits-splits + metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-stage-1-tuner + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters-2 + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters-2 + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + taskInfo: + name: automl-tabular-cv-trainer-2 + automl-tabular-ensemble-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble-2 + dependentTasks: + - automl-tabular-cv-trainer-2 + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-cv-trainer-2 + warmup_data: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble-2 + automl-tabular-infra-validator-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator-2 + dependentTasks: + - automl-tabular-ensemble-2 + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + taskInfo: + name: automl-tabular-infra-validator-2 + automl-tabular-stage-1-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-stage-1-tuner + dependentTasks: + - calculate-training-parameters-2 + inputs: + artifacts: + feature_ranking: + componentInputArtifact: pipelinechannel--tabular-feature-ranking-and-selection-feature_ranking + materialized_eval_split: + componentInputArtifact: pipelinechannel--automl-tabular-transform-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--automl-tabular-transform-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + taskOutputParameter: + outputParameterKey: stage_1_num_selected_trials + producerTask: calculate-training-parameters-2 + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + taskOutputParameter: + outputParameterKey: reduce_search_space_mode + producerTask: calculate-training-parameters-2 + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + tune_feature_selection_rate: + componentInputParameter: pipelinechannel--apply_feature_selection_tuning + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-tabular-stage-1-tuner + bool-identity-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity-2 + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity-2 + bool-identity-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity-3 + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_distillation + taskInfo: + name: bool-identity-3 + calculate-training-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters-2 + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: false + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters-2 + condition-5: + componentRef: + name: comp-condition-5 + dependentTasks: + - automl-tabular-ensemble-2 + - bool-identity-2 + - bool-identity-3 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-2 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-2 + pipelinechannel--bool-identity-2-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-2 + pipelinechannel--bool-identity-3-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-3 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: no-distill + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-3-Output'] + == 'false' + condition-7: + componentRef: + name: comp-condition-7 + dependentTasks: + - automl-tabular-ensemble-2 + - bool-identity-2 + - bool-identity-3 + - calculate-training-parameters-2 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + pipelinechannel--purge-unused-features-output_metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + pipelinechannel--tabular-stats-and-example-gen-eval_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + pipelinechannel--tabular-stats-and-example-gen-test_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-test_split + pipelinechannel--tabular-stats-and-example-gen-train_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-train_split + parameters: + pipelinechannel--bool-identity-2-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-2 + pipelinechannel--bool-identity-3-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-3 + pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours: + taskOutputParameter: + outputParameterKey: distill_stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode: + taskOutputParameter: + outputParameterKey: reduce_search_space_mode + producerTask: calculate-training-parameters-2 + pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + taskInfo: + name: is-distill + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-3-Output'] + == 'true' + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-transform-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-transform-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-transform-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--merge-materialized-splits-splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--purge-unused-features-output_metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-feature-ranking-and-selection-feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + parameters: + pipelinechannel--apply_feature_selection_tuning: + parameterType: BOOLEAN + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-5: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-6 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-6 + tasks: + condition-6: + componentRef: + name: comp-condition-6 + dependentTasks: + - model-upload-2 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + pipelinechannel--model-upload-2-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-2 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + pipelinechannel--bool-identity-2-Output: + componentInputParameter: pipelinechannel--bool-identity-2-Output + pipelinechannel--bool-identity-3-Output: + componentInputParameter: pipelinechannel--bool-identity-3-Output + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-2-Output'] + == 'true' + model-upload-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-6: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-2 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation-2 + tasks: + feature-attribution-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-2 + dependentTasks: + - model-batch-explanation-2 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-2 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-2 + model-batch-explanation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-2 + model-batch-predict-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-2 + model-evaluation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-2 + dependentTasks: + - model-batch-predict-2 + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-2 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation-2 + model-evaluation-import-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-2 + dependentTasks: + - feature-attribution-2 + - model-evaluation-2 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-2 + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-2 + model: + componentInputArtifact: pipelinechannel--model-upload-2-model + parameters: + dataset_paths: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + dataset_type: + runtimeValue: + constant: tf-record + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-2-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-7: + dag: + outputs: + artifacts: + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-8 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-8 + tasks: + automl-tabular-ensemble-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble-3 + dependentTasks: + - automl-tabular-stage-1-tuner-2 + - automl-tabular-transform-2 + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform-2 + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-stage-1-tuner-2 + warmup_data: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble-3 + automl-tabular-infra-validator-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator-3 + dependentTasks: + - automl-tabular-ensemble-3 + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + taskInfo: + name: automl-tabular-infra-validator-3 + automl-tabular-stage-1-tuner-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-stage-1-tuner-2 + dependentTasks: + - automl-tabular-transform-2 + inputs: + artifacts: + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: automl-tabular-transform-2 + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: automl-tabular-transform-2 + metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform-2 + parameters: + deadline_hours: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours + disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + runtimeValue: + constant: 1.0 + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_distillation: + runtimeValue: + constant: true + single_run_max_secs: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-tabular-stage-1-tuner-2 + automl-tabular-transform-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-transform-2 + dependentTasks: + - write-bp-result-path + - write-bp-result-path-2 + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + eval_split: + taskOutputArtifact: + outputArtifactKey: result + producerTask: write-bp-result-path-2 + metadata: + componentInputArtifact: pipelinechannel--purge-unused-features-output_metadata + test_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-test_split + train_split: + taskOutputArtifact: + outputArtifactKey: result + producerTask: write-bp-result-path + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-transform-2 + condition-8: + componentRef: + name: comp-condition-8 + dependentTasks: + - automl-tabular-ensemble-3 + - model-upload-3 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-3 + pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + pipelinechannel--model-upload-3-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-3 + parameters: + pipelinechannel--automl-tabular-ensemble-3-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-3 + pipelinechannel--bool-identity-2-Output: + componentInputParameter: pipelinechannel--bool-identity-2-Output + pipelinechannel--bool-identity-3-Output: + componentInputParameter: pipelinechannel--bool-identity-3-Output + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-2-Output'] + == 'true' + model-batch-predict-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-3 + dependentTasks: + - read-input-uri + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + taskOutputParameter: + outputParameterKey: Output + producerTask: read-input-uri + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-train-split + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: tf-record + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-3 + model-batch-predict-4: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-4 + dependentTasks: + - read-input-uri-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + taskOutputParameter: + outputParameterKey: Output + producerTask: read-input-uri-2 + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-eval-split + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: tf-record + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-4 + model-upload-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-3 + dependentTasks: + - automl-tabular-ensemble-3 + - automl-tabular-infra-validator-3 + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-3 + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + parameters: + display_name: + runtimeValue: + constant: automl-tabular-distill-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-3 + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-3 + read-input-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-read-input-uri + inputs: + artifacts: + split_uri: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-train_split + taskInfo: + name: read-input-uri + read-input-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-read-input-uri-2 + inputs: + artifacts: + split_uri: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + taskInfo: + name: read-input-uri-2 + write-bp-result-path: + cachingOptions: + enableCache: true + componentRef: + name: comp-write-bp-result-path + dependentTasks: + - model-batch-predict-3 + inputs: + artifacts: + bp_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-3 + taskInfo: + name: write-bp-result-path + write-bp-result-path-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-write-bp-result-path-2 + dependentTasks: + - model-batch-predict-4 + inputs: + artifacts: + bp_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-4 + taskInfo: + name: write-bp-result-path-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--purge-unused-features-output_metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode: + parameterType: STRING + pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-8: + dag: + outputs: + artifacts: + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-3 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation-3 + tasks: + feature-attribution-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-3 + dependentTasks: + - model-batch-explanation-3 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-3 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-3 + model-batch-explanation-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-3 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-3-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-3 + model-batch-predict-5: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-5 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-5 + model-evaluation-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-3 + dependentTasks: + - model-batch-predict-5 + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-5 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation-3 + model-evaluation-import-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-3 + dependentTasks: + - feature-attribution-3 + - model-evaluation-3 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-3 + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-3 + model: + componentInputArtifact: pipelinechannel--model-upload-3-model + parameters: + dataset_paths: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + dataset_type: + runtimeValue: + constant: tf-record + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import-3 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-3-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-3-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-4 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-4 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-2 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-4 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-4 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + automl-tabular-transform: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-transform + dependentTasks: + - purge-unused-features + - tabular-stats-and-example-gen + inputs: + artifacts: + dataset_schema: + taskOutputArtifact: + outputArtifactKey: dataset_schema + producerTask: tabular-stats-and-example-gen + eval_split: + taskOutputArtifact: + outputArtifactKey: eval_split + producerTask: tabular-stats-and-example-gen + metadata: + taskOutputArtifact: + outputArtifactKey: output_metadata + producerTask: purge-unused-features + test_split: + taskOutputArtifact: + outputArtifactKey: test_split + producerTask: tabular-stats-and-example-gen + train_split: + taskOutputArtifact: + outputArtifactKey: train_split + producerTask: tabular-stats-and-example-gen + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-transform + check-if-binary-classification: + cachingOptions: + enableCache: true + componentRef: + name: comp-check-if-binary-classification + dependentTasks: + - tabular-stats-and-example-gen + inputs: + artifacts: + example_gen_metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: tabular-stats-and-example-gen + taskInfo: + name: check-if-binary-classification + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - automl-tabular-transform + - merge-materialized-splits + - purge-unused-features + - string-not-empty + - tabular-stats-and-example-gen + inputs: + artifacts: + pipelinechannel--automl-tabular-transform-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform + pipelinechannel--merge-materialized-splits-splits: + taskOutputArtifact: + outputArtifactKey: splits + producerTask: merge-materialized-splits + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--purge-unused-features-output_metadata: + taskOutputArtifact: + outputArtifactKey: output_metadata + producerTask: purge-unused-features + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + taskOutputArtifact: + outputArtifactKey: dataset_schema + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-eval_split: + taskOutputArtifact: + outputArtifactKey: eval_split + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: tabular-stats-and-example-gen + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_distillation: + componentInputParameter: pipelinechannel--run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + taskOutputParameter: + outputParameterKey: downsampled_test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + taskOutputParameter: + outputParameterKey: test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_not_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'true' + condition-4: + componentRef: + name: comp-condition-4 + dependentTasks: + - automl-tabular-transform + - merge-materialized-splits + - purge-unused-features + - string-not-empty + - tabular-feature-ranking-and-selection + - tabular-stats-and-example-gen + inputs: + artifacts: + pipelinechannel--automl-tabular-transform-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: automl-tabular-transform + pipelinechannel--automl-tabular-transform-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: automl-tabular-transform + pipelinechannel--automl-tabular-transform-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform + pipelinechannel--merge-materialized-splits-splits: + taskOutputArtifact: + outputArtifactKey: splits + producerTask: merge-materialized-splits + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--purge-unused-features-output_metadata: + taskOutputArtifact: + outputArtifactKey: output_metadata + producerTask: purge-unused-features + pipelinechannel--tabular-feature-ranking-and-selection-feature_ranking: + taskOutputArtifact: + outputArtifactKey: feature_ranking + producerTask: tabular-feature-ranking-and-selection + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + taskOutputArtifact: + outputArtifactKey: dataset_schema + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-eval_split: + taskOutputArtifact: + outputArtifactKey: eval_split + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-test_split: + taskOutputArtifact: + outputArtifactKey: test_split + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-train_split: + taskOutputArtifact: + outputArtifactKey: train_split + producerTask: tabular-stats-and-example-gen + parameters: + pipelinechannel--apply_feature_selection_tuning: + componentInputParameter: pipelinechannel--apply_feature_selection_tuning + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--model_display_name: + componentInputParameter: pipelinechannel--model_display_name + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_distillation: + componentInputParameter: pipelinechannel--run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + taskOutputParameter: + outputParameterKey: downsampled_test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + taskOutputParameter: + outputParameterKey: test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + taskInfo: + name: stage_1_tuning_result_artifact_uri_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'false' + merge-materialized-splits: + cachingOptions: + enableCache: true + componentRef: + name: comp-merge-materialized-splits + dependentTasks: + - automl-tabular-transform + inputs: + artifacts: + split_0: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: automl-tabular-transform + split_1: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: automl-tabular-transform + taskInfo: + name: merge-materialized-splits + purge-unused-features: + cachingOptions: + enableCache: true + componentRef: + name: comp-purge-unused-features + dependentTasks: + - tabular-feature-ranking-and-selection + - tabular-stats-and-example-gen + inputs: + artifacts: + selected_features: + taskOutputArtifact: + outputArtifactKey: selected_features + producerTask: tabular-feature-ranking-and-selection + unpurged_metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: tabular-stats-and-example-gen + taskInfo: + name: purge-unused-features + string-not-empty: + cachingOptions: + enableCache: true + componentRef: + name: comp-string-not-empty + inputs: + parameters: + value: + runtimeValue: + constant: '' + taskInfo: + name: string-not-empty + tabular-feature-ranking-and-selection: + cachingOptions: + enableCache: true + componentRef: + name: comp-tabular-feature-ranking-and-selection + dependentTasks: + - check-if-binary-classification + - tabular-stats-and-example-gen + inputs: + artifacts: + data_source: + taskOutputArtifact: + outputArtifactKey: train_split + producerTask: tabular-stats-and-example-gen + parameters: + binary_classification: + taskOutputParameter: + outputParameterKey: Output + producerTask: check-if-binary-classification + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_column_name: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: tabular-feature-ranking-and-selection + tabular-stats-and-example-gen: + cachingOptions: + enableCache: true + componentRef: + name: comp-tabular-stats-and-example-gen + inputs: + parameters: + additional_experiments_json: + componentInputParameter: pipelinechannel--additional_experiments + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + optimization_objective_precision_value: + componentInputParameter: pipelinechannel--optimization_objective_precision_value + optimization_objective_recall_value: + componentInputParameter: pipelinechannel--optimization_objective_recall_value + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + quantiles: + componentInputParameter: pipelinechannel--quantiles + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column_name: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + transformations: + runtimeValue: + constant: '[]' + transformations_path: + componentInputParameter: pipelinechannel--transformations + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column_name: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: tabular-stats-and-example-gen + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--additional_experiments: + parameterType: STRUCT + pipelinechannel--apply_feature_selection_tuning: + parameterType: BOOLEAN + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--data_source_csv_filenames: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--enable_probabilistic_inference: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--model_display_name: + parameterType: STRING + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--optimization_objective_precision_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--optimization_objective_recall_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stats_and_example_gen_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--stats_and_example_gen_dataflow_machine_type: + parameterType: STRING + pipelinechannel--stats_and_example_gen_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--transformations: + parameterType: STRING + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-attribution: + executorLabel: exec-feature-attribution + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-2: + executorLabel: exec-feature-attribution-2 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-3: + executorLabel: exec-feature-attribution-3 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-merge-materialized-splits: + executorLabel: exec-merge-materialized-splits + inputDefinitions: + artifacts: + split_0: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The first materialized split. + split_1: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The second materialized split. + outputDefinitions: + artifacts: + splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-model-batch-explanation: + executorLabel: exec-model-batch-explanation + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-2: + executorLabel: exec-model-batch-explanation-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-3: + executorLabel: exec-model-batch-explanation-3 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-2: + executorLabel: exec-model-batch-predict-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-3: + executorLabel: exec-model-batch-predict-3 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-4: + executorLabel: exec-model-batch-predict-4 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-5: + executorLabel: exec-model-batch-predict-5 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-2: + executorLabel: exec-model-evaluation-2 + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-3: + executorLabel: exec-model-evaluation-3 + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-2: + executorLabel: exec-model-evaluation-import-2 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-3: + executorLabel: exec-model-evaluation-import-3 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-2: + executorLabel: exec-model-upload-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-3: + executorLabel: exec-model-upload-3 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-purge-unused-features: + executorLabel: exec-purge-unused-features + inputDefinitions: + artifacts: + selected_features: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: selected feature names separated by comma. + unpurged_metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: metadata generated by example gen. + outputDefinitions: + artifacts: + output_metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-read-input-uri: + executorLabel: exec-read-input-uri + inputDefinitions: + artifacts: + split_uri: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: Tbe path to the file that contains Dataset data. + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-read-input-uri-2: + executorLabel: exec-read-input-uri-2 + inputDefinitions: + artifacts: + split_uri: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: Tbe path to the file that contains Dataset data. + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-string-not-empty: + executorLabel: exec-string-not-empty + inputDefinitions: + parameters: + value: + description: String value to be checked. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-tabular-feature-ranking-and-selection: + executorLabel: exec-tabular-feature-ranking-and-selection + inputDefinitions: + artifacts: + data_source: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + parameters: + algorithm: + defaultValue: AMI + isOptional: true + parameterType: STRING + binary_classification: + defaultValue: 'false' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption key. + + If this is set, then all resources will be encrypted with the provided + + encryption key. data_source(Dataset): The input dataset artifact which + + references csv, BigQuery, or TF Records. target_column_name(str): Target + + column name of the input dataset.' + isOptional: true + parameterType: STRING + location: + description: 'Location for running the feature selection. If not set, + + default to us-central1.' + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'number of features to select by the + + algorithm. If not set, default to 1000.' + isOptional: true + parameterType: NUMBER_INTEGER + prediction_type: + defaultValue: unknown + isOptional: true + parameterType: STRING + project: + description: Project to run feature selection. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + target_column_name: + parameterType: STRING + outputDefinitions: + artifacts: + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: the dictionary of feature names and feature ranking values. + selected_features: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: A json array of selected feature names. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-tabular-stats-and-example-gen: + executorLabel: exec-tabular-stats-and-example-gen + inputDefinitions: + parameters: + additional_experiments: + defaultValue: '' + isOptional: true + parameterType: STRING + additional_experiments_json: + defaultValue: {} + isOptional: true + parameterType: STRUCT + data_source_bigquery_table_path: + defaultValue: '' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + enable_probabilistic_inference: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: 'Location for running dataset statistics and example + + generation.' + parameterType: STRING + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + predefined_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_type: + description: 'The prediction type. Supported values: + + "classification", "regression".' + parameterType: STRING + project: + description: 'Project to run dataset statistics and example + + generation.' + parameterType: STRING + quantiles: + defaultValue: [] + isOptional: true + parameterType: LIST + request_type: + defaultValue: COLUMN_STATS_ONLY + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + stratified_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + target_column_name: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + timestamp_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + transformations: + description: 'Quote escaped JSON string for transformations. Each + + transformation will apply transform function to given input column. And + + the result will be used for training. When creating transformation for + + BigQuery Struct column, the column should be flattened using "." as the + + delimiter.' + parameterType: STRING + transformations_path: + defaultValue: '' + description: 'Path to a GCS file containing JSON + + string for transformations.' + isOptional: true + parameterType: STRING + validation_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column_name: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The eval split. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The instance baseline used to calculate explanations. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The test split. + train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The train split. + parameters: + downsampled_test_split_json: + description: The downsampled test split JSON object. + parameterType: LIST + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + test_split_json: + description: The test split JSON object. + parameterType: LIST + comp-write-bp-result-path: + executorLabel: exec-write-bp-result-path + inputDefinitions: + artifacts: + bp_job: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The batch prediction job artifact. + outputDefinitions: + artifacts: + result: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + comp-write-bp-result-path-2: + executorLabel: exec-write-bp-result-path-2 + inputDefinitions: + artifacts: + bp_job: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The batch prediction job artifact. + outputDefinitions: + artifacts: + result: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-automl-tabular-cv-trainer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-cv-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--valid_trials_completed_threshold=0.7\", \"--num_selected_trials=", + "{{$.inputs.parameters[''num_selected_trials'']}}", "\", \"--num_selected_features=", + "{{$.inputs.parameters[''num_selected_features'']}}", "\", \"--lro_job_info=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_cv_splits=", + "{{$.inputs.artifacts[''materialized_cv_splits''].uri}}", "\", \"--tuning_result_input_path=", + "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_custom_job=true\", + \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-cv-trainer-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-cv-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--valid_trials_completed_threshold=0.7\", \"--num_selected_trials=", + "{{$.inputs.parameters[''num_selected_trials'']}}", "\", \"--num_selected_features=", + "{{$.inputs.parameters[''num_selected_features'']}}", "\", \"--lro_job_info=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_cv_splits=", + "{{$.inputs.artifacts[''materialized_cv_splits''].uri}}", "\", \"--tuning_result_input_path=", + "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_custom_job=true\", + \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble-3: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-infra-validator: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-infra-validator-2: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-infra-validator-3: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-stage-1-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", + "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", + "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--num_selected_features=", "{{$.inputs.parameters[''num_selected_features'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=", + "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=", + "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=", + "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\", + \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-stage-1-tuner-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", + "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", + "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--num_selected_features=", "{{$.inputs.parameters[''num_selected_features'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=", + "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=", + "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=", + "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\", + \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-transform: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", + "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", + \"--materialized_splits_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=", + "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=", + "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}", + "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\", + \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}", + "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", + \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", + \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", + "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-transform-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", + "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", + \"--materialized_splits_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=", + "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=", + "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}", + "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\", + \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}", + "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", + \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", + \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", + "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-bool-identity-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-bool-identity-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-calculate-training-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n run_distillation: bool,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_num_selected_trials',\ + \ int),\n ('stage_1_single_run_max_secs', int),\n ('stage_2_deadline_hours',\ + \ float),\n ('stage_2_single_run_max_secs', int),\n ('distill_stage_1_deadline_hours',\ + \ float),\n ('reduce_search_space_mode', str),\n ],\n):\n \"\"\ + \"Calculates training parameters.\n\n Args:\n stage_1_num_parallel_trials:\ + \ Number of parallel trails for stage 1.\n train_budget_milli_node_hours:\ + \ The train budget of creating this model,\n expressed in milli node\ + \ hours i.e. 1,000 value in this field means 1 node\n hour.\n stage_2_num_parallel_trials:\ + \ Number of parallel trails for stage 2.\n run_distillation: Whether\ + \ to run distill in the training pipeline.\n is_skip_architecture_search:\ + \ If component is being called in the\n skip_architecture_search pipeline.\n\ + \ fast_testing: Internal flag used for presubmit tests.\n\n Returns:\n\ + \ stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\ + \ stage_1_num_selected_trials: Number of selected trails for stage\ + \ 1.\n stage_1_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 1\n training trial.\n stage_2_deadline_hours:\ + \ Maximum number of hours to run stage 2.\n stage_2_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 2\n training\ + \ trial.\n distill_stage_1_deadline_hours: Maximum number of hours\ + \ to run stage 1 for\n the model distillation.\n reduce_search_space_mode:\ + \ The reduce search space mode. Possible values:\n minimal, regular,\ + \ full.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ num_folds = 5\n distill_total_trials = 100\n\n stage_1_deadline_hours\ + \ = -1.0\n stage_1_num_selected_trials = -1\n stage_1_single_run_max_secs\ + \ = -1\n stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs =\ + \ -1\n distill_stage_1_deadline_hours = 1.0\n reduce_search_space_mode\ + \ = 'regular'\n\n if is_skip_architecture_search:\n stage_2_deadline_hours\ + \ = train_budget_milli_node_hours / 1000.0\n stage_2_single_run_max_secs\ + \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n else:\n hours = float(train_budget_milli_node_hours)\ + \ / 1000.0\n multiplier = stage_1_num_parallel_trials * hours / 500.0\n\ + \ stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\ + \ phase_2_rounds = int(\n math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\ + \ + 0.5\n )\n if phase_2_rounds < 1:\n phase_2_rounds = 1\n\n\ + \ # All of magic number \"1.3\" above is because the trial doesn't\n\ + \ # always finish in time_per_trial. 1.3 is an empirical safety margin\ + \ here.\n stage_1_deadline_secs = int(\n hours * 3600.0 - 1.3\ + \ * stage_1_single_run_max_secs * phase_2_rounds\n )\n\n if stage_1_deadline_secs\ + \ < hours * 3600.0 * 0.5:\n stage_1_deadline_secs = int(hours * 3600.0\ + \ * 0.5)\n # Phase 1 deadline is the same as phase 2 deadline in this\ + \ case. Phase 2\n # can't finish in time after the deadline is cut,\ + \ so adjust the time per\n # trial to meet the deadline.\n stage_1_single_run_max_secs\ + \ = int(\n stage_1_deadline_secs / (1.3 * phase_2_rounds)\n \ + \ )\n\n reduce_search_space_mode = 'minimal'\n if multiplier > 2:\n\ + \ reduce_search_space_mode = 'regular'\n if multiplier > 4:\n \ + \ reduce_search_space_mode = 'full'\n\n # Stage 2 number of trials\ + \ is stage_1_num_selected_trials *\n # num_folds, which should be equal\ + \ to phase_2_rounds *\n # stage_2_num_parallel_trials. Use this information\ + \ to calculate\n # stage_1_num_selected_trials:\n stage_1_num_selected_trials\ + \ = int(\n phase_2_rounds * stage_2_num_parallel_trials / num_folds\n\ + \ )\n stage_1_deadline_hours = stage_1_deadline_secs / 3600.0\n\n\ + \ stage_2_deadline_hours = hours - stage_1_deadline_hours\n stage_2_single_run_max_secs\ + \ = stage_1_single_run_max_secs\n\n if run_distillation:\n # All\ + \ of magic number \"1.3\" above is because the trial doesn't always\n \ + \ # finish in time_per_trial. 1.3 is an empirical safety margin here.\n\ + \ distill_stage_1_deadline_hours = (\n math.ceil(float(distill_total_trials)\ + \ / stage_1_num_parallel_trials)\n * stage_1_single_run_max_secs\n\ + \ * 1.3\n / 3600.0\n )\n\n if fast_testing:\n \ + \ distill_stage_1_deadline_hours = 0.2\n stage_1_deadline_hours = 0.2\n\ + \ stage_1_single_run_max_secs = 1\n stage_2_deadline_hours = 0.2\n\ + \ stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_num_selected_trials',\n 'stage_1_single_run_max_secs',\n\ + \ 'stage_2_deadline_hours',\n 'stage_2_single_run_max_secs',\n\ + \ 'distill_stage_1_deadline_hours',\n 'reduce_search_space_mode',\n\ + \ ],\n )(\n stage_1_deadline_hours,\n stage_1_num_selected_trials,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ + \ reduce_search_space_mode,\n )\n\n" + image: python:3.7 + exec-calculate-training-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n run_distillation: bool,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_num_selected_trials',\ + \ int),\n ('stage_1_single_run_max_secs', int),\n ('stage_2_deadline_hours',\ + \ float),\n ('stage_2_single_run_max_secs', int),\n ('distill_stage_1_deadline_hours',\ + \ float),\n ('reduce_search_space_mode', str),\n ],\n):\n \"\"\ + \"Calculates training parameters.\n\n Args:\n stage_1_num_parallel_trials:\ + \ Number of parallel trails for stage 1.\n train_budget_milli_node_hours:\ + \ The train budget of creating this model,\n expressed in milli node\ + \ hours i.e. 1,000 value in this field means 1 node\n hour.\n stage_2_num_parallel_trials:\ + \ Number of parallel trails for stage 2.\n run_distillation: Whether\ + \ to run distill in the training pipeline.\n is_skip_architecture_search:\ + \ If component is being called in the\n skip_architecture_search pipeline.\n\ + \ fast_testing: Internal flag used for presubmit tests.\n\n Returns:\n\ + \ stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\ + \ stage_1_num_selected_trials: Number of selected trails for stage\ + \ 1.\n stage_1_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 1\n training trial.\n stage_2_deadline_hours:\ + \ Maximum number of hours to run stage 2.\n stage_2_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 2\n training\ + \ trial.\n distill_stage_1_deadline_hours: Maximum number of hours\ + \ to run stage 1 for\n the model distillation.\n reduce_search_space_mode:\ + \ The reduce search space mode. Possible values:\n minimal, regular,\ + \ full.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ num_folds = 5\n distill_total_trials = 100\n\n stage_1_deadline_hours\ + \ = -1.0\n stage_1_num_selected_trials = -1\n stage_1_single_run_max_secs\ + \ = -1\n stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs =\ + \ -1\n distill_stage_1_deadline_hours = 1.0\n reduce_search_space_mode\ + \ = 'regular'\n\n if is_skip_architecture_search:\n stage_2_deadline_hours\ + \ = train_budget_milli_node_hours / 1000.0\n stage_2_single_run_max_secs\ + \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n else:\n hours = float(train_budget_milli_node_hours)\ + \ / 1000.0\n multiplier = stage_1_num_parallel_trials * hours / 500.0\n\ + \ stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\ + \ phase_2_rounds = int(\n math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\ + \ + 0.5\n )\n if phase_2_rounds < 1:\n phase_2_rounds = 1\n\n\ + \ # All of magic number \"1.3\" above is because the trial doesn't\n\ + \ # always finish in time_per_trial. 1.3 is an empirical safety margin\ + \ here.\n stage_1_deadline_secs = int(\n hours * 3600.0 - 1.3\ + \ * stage_1_single_run_max_secs * phase_2_rounds\n )\n\n if stage_1_deadline_secs\ + \ < hours * 3600.0 * 0.5:\n stage_1_deadline_secs = int(hours * 3600.0\ + \ * 0.5)\n # Phase 1 deadline is the same as phase 2 deadline in this\ + \ case. Phase 2\n # can't finish in time after the deadline is cut,\ + \ so adjust the time per\n # trial to meet the deadline.\n stage_1_single_run_max_secs\ + \ = int(\n stage_1_deadline_secs / (1.3 * phase_2_rounds)\n \ + \ )\n\n reduce_search_space_mode = 'minimal'\n if multiplier > 2:\n\ + \ reduce_search_space_mode = 'regular'\n if multiplier > 4:\n \ + \ reduce_search_space_mode = 'full'\n\n # Stage 2 number of trials\ + \ is stage_1_num_selected_trials *\n # num_folds, which should be equal\ + \ to phase_2_rounds *\n # stage_2_num_parallel_trials. Use this information\ + \ to calculate\n # stage_1_num_selected_trials:\n stage_1_num_selected_trials\ + \ = int(\n phase_2_rounds * stage_2_num_parallel_trials / num_folds\n\ + \ )\n stage_1_deadline_hours = stage_1_deadline_secs / 3600.0\n\n\ + \ stage_2_deadline_hours = hours - stage_1_deadline_hours\n stage_2_single_run_max_secs\ + \ = stage_1_single_run_max_secs\n\n if run_distillation:\n # All\ + \ of magic number \"1.3\" above is because the trial doesn't always\n \ + \ # finish in time_per_trial. 1.3 is an empirical safety margin here.\n\ + \ distill_stage_1_deadline_hours = (\n math.ceil(float(distill_total_trials)\ + \ / stage_1_num_parallel_trials)\n * stage_1_single_run_max_secs\n\ + \ * 1.3\n / 3600.0\n )\n\n if fast_testing:\n \ + \ distill_stage_1_deadline_hours = 0.2\n stage_1_deadline_hours = 0.2\n\ + \ stage_1_single_run_max_secs = 1\n stage_2_deadline_hours = 0.2\n\ + \ stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_num_selected_trials',\n 'stage_1_single_run_max_secs',\n\ + \ 'stage_2_deadline_hours',\n 'stage_2_single_run_max_secs',\n\ + \ 'distill_stage_1_deadline_hours',\n 'reduce_search_space_mode',\n\ + \ ],\n )(\n stage_1_deadline_hours,\n stage_1_num_selected_trials,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ + \ reduce_search_space_mode,\n )\n\n" + image: python:3.7 + exec-check-if-binary-classification: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _check_if_binary_classification + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _check_if_binary_classification(\n example_gen_metadata: dsl.InputPath('TabularExampleGenMetadata'),\n\ + ) -> str:\n \"\"\"Construct Dataset based on the batch prediction job.\n\ + \n Args:\n example_gen_metadata: metadata generated by example gen.\n\ + \n Returns:\n \"true\" if binary classification, \"false\" otherwise.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ with open(example_gen_metadata, 'r') as f:\n metadata_path = f.read()\n\ + \ metadata = json.loads(metadata_path)\n return str(metadata['objective']\ + \ == 'binary_classification').lower()\n\n" + image: python:3.7 + exec-feature-attribution: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-2: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-3: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-importer: + importer: + artifactUri: + constant: '' + typeSchema: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + exec-merge-materialized-splits: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _merge_materialized_splits + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _merge_materialized_splits(\n split_0: dsl.InputPath('MaterializedSplit'),\n\ + \ split_1: dsl.InputPath('MaterializedSplit'),\n splits: dsl.OutputPath('MaterializedSplit'),\n\ + ):\n \"\"\"Merge two materialized splits.\n\n Args:\n split_0: The\ + \ first materialized split.\n split_1: The second materialized split.\n\ + \ splits: The merged materialized split.\n \"\"\"\n with open(split_0,\ + \ 'r') as f:\n split_0_content = f.read()\n with open(split_1, 'r')\ + \ as f:\n split_1_content = f.read()\n with open(splits, 'w') as f:\n\ + \ f.write(','.join([split_0_content, split_1_content]))\n\n" + image: python:3.7 + exec-model-batch-explanation: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-3: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-3: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-4: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-5: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-2: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-3: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-2: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-3: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-2: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-3: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-purge-unused-features: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _purge_unused_features + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _purge_unused_features(\n unpurged_metadata: dsl.InputPath('TabularExampleGenMetadata'),\n\ + \ selected_features: dsl.InputPath('SelectedFeatures'),\n output_metadata:\ + \ dsl.OutputPath('TabularExampleGenMetadata'),\n):\n \"\"\"Purge features\ + \ from metadata if not included in selected features.\n\n Args:\n unpurged_metadata:\ + \ metadata generated by example gen.\n selected_features: selected feature\ + \ names separated by comma.\n output_metadata: purged metadata.\n \"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ with open(unpurged_metadata, 'r') as f:\n metadata_path = f.read()\n\ + \ metadata = json.loads(metadata_path)\n\n with open(selected_features,\ + \ 'r') as f:\n selected_features_path = f.read()\n features = json.loads(selected_features_path)\n\ + \n train_spec = metadata['train_spec']\n\n features_set = set(features)\n\ + \n purged_transformation_list = []\n for transformation in train_spec['transformations']:\n\ + \ if 'numeric' in transformation:\n if transformation['numeric']['column_name']\ + \ in features_set:\n purged_transformation_list.append(transformation)\n\ + \ elif 'categorical' in transformation:\n if transformation['categorical']['column_name']\ + \ in features_set:\n purged_transformation_list.append(transformation)\n\ + \ elif 'timestamp' in transformation:\n if transformation['timestamp']['column_name']\ + \ in features_set:\n purged_transformation_list.append(transformation)\n\ + \ elif 'text' in transformation:\n if transformation['text']['column_name']\ + \ in features_set:\n purged_transformation_list.append(transformation)\n\ + \ elif 'repeated_numeric' in transformation:\n if transformation['repeated_numeric']['column_name']\ + \ in features_set:\n purged_transformation_list.append(transformation)\n\ + \ elif 'repeated_categorical' in transformation:\n if transformation['repeated_categorical']['column_name']\ + \ in features_set:\n purged_transformation_list.append(transformation)\n\ + \ elif 'repeated_text' in transformation:\n if transformation['repeated_text']['column_name']\ + \ in features_set:\n purged_transformation_list.append(transformation)\n\ + \ else:\n raise ValueError(f'unsupported transformation: {transformation}')\n\ + \n train_spec['transformations'] = purged_transformation_list\n metadata['train_spec']\ + \ = train_spec\n\n with open(output_metadata, 'w') as f:\n f.write(json.dumps(metadata))\n\ + \n" + image: python:3.7 + exec-read-input-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _read_input_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _read_input_uri(\n split_uri: dsl.InputPath('Dataset'),\n\ + ) -> list: # Required by KFP; pylint:disable=g-bare-generic\n \"\"\"Construct\ + \ Dataset based on the batch prediction job.\n\n Args:\n split_uri:\ + \ Tbe path to the file that contains Dataset data.\n\n Returns:\n The\ + \ list of string that represents the batch prediction input files.\n \"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ + \ return data_source['tf_record_data_source']['file_patterns']\n\n" + image: python:3.7 + exec-read-input-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _read_input_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _read_input_uri(\n split_uri: dsl.InputPath('Dataset'),\n\ + ) -> list: # Required by KFP; pylint:disable=g-bare-generic\n \"\"\"Construct\ + \ Dataset based on the batch prediction job.\n\n Args:\n split_uri:\ + \ Tbe path to the file that contains Dataset data.\n\n Returns:\n The\ + \ list of string that represents the batch prediction input files.\n \"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ + \ return data_source['tf_record_data_source']['file_patterns']\n\n" + image: python:3.7 + exec-string-not-empty: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _string_not_empty + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _string_not_empty(value: str) -> str:\n \"\"\"Check if the input\ + \ string value is not empty.\n\n Args:\n value: String value to be checked.\n\ + \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ + \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ + \ \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-tabular-feature-ranking-and-selection: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"tabular-feature-selection-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"feature_selection\", \"--data_source=", "{{$.inputs.artifacts[''data_source''].uri}}", + "\", \"--target_column=", "{{$.inputs.parameters[''target_column_name'']}}", + "\", \"--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}", + "\", \"--binary_classification=", "{{$.inputs.parameters[''binary_classification'']}}", + "\", \"--algorithm=", "{{$.inputs.parameters[''algorithm'']}}", "\", \"--feature_selection_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection/\", + \"--job_name=tabular-feature-selection-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", + \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", + \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", + "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}", + "\", \"--feature_selection_result_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}", + "\", \"--selected_features_path=", "{{$.outputs.artifacts[''selected_features''].uri}}", + "\", \"--parse_json=true\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-tabular-stats-and-example-gen: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"tabular-stats-and-example-gen-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"stats_generator\",", "\"--train_spec={\\\"prediction_type\\\": + \\\"", "{{$.inputs.parameters[''prediction_type'']}}", "\\\", \\\"target_column\\\": + \\\"", "{{$.inputs.parameters[''target_column_name'']}}", "\\\", \\\"optimization_objective\\\": + \\\"", "{{$.inputs.parameters[''optimization_objective'']}}", "\\\", \\\"weight_column_name\\\": + \\\"", "{{$.inputs.parameters[''weight_column_name'']}}", "\\\", \\\"transformations\\\": + ", "{{$.inputs.parameters[''transformations'']}}", ", \\\"quantiles\\\": + ", "{{$.inputs.parameters[''quantiles'']}}", ", \\\"enable_probabilistic_inference\\\": + ", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}", "}\", + \"--transformations_override_path=", "{{$.inputs.parameters[''transformations_path'']}}", + "\", \"--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}", + "\", \"--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}", + "\", \"--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}", + "\", \"--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}", + "\", \"--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}", + "\", \"--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}", + "\", \"--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}", + "\", \"--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}", + "\", \"--target_column=", "{{$.inputs.parameters[''target_column_name'']}}", + "\", \"--request_type=", "{{$.inputs.parameters[''request_type'']}}", "\", + \"--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}", + "\", \"--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}", + "\", \"--example_gen_gcs_output_prefix=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/example_gen_output\", + \"--dataset_stats_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/stats/\", + \"--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}", + "\", \"--dataset_schema_path=", "{{$.outputs.artifacts[''dataset_schema''].uri}}", + "\", \"--job_name=tabular-stats-and-example-gen-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", + \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", + \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", + "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"--is_distill=", "{{$.inputs.parameters[''run_distillation'']}}", + "\", \"--additional_experiments=", "{{$.inputs.parameters[''additional_experiments'']}}", + "\", \"--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}", "\", + \"--train_split=", "{{$.outputs.artifacts[''train_split''].uri}}", "\", + \"--eval_split=", "{{$.outputs.artifacts[''eval_split''].uri}}", "\", \"--test_split=", + "{{$.outputs.artifacts[''test_split''].uri}}", "\", \"--test_split_for_batch_prediction_component=", + "{{$.outputs.parameters[''test_split_json''].output_file}}", "\", \"--downsampled_test_split_for_batch_prediction_component=", + "{{$.outputs.parameters[''downsampled_test_split_json''].output_file}}", + "\", \"--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--parse_json=true\", \"--generate_additional_downsample_test_split=true\", + \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-write-bp-result-path: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _write_bp_result_path + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _write_bp_result_path(\n bp_job: dsl.Input[dsl.Artifact],\n\ + \ result: dsl.OutputPath('Dataset'),\n):\n \"\"\"Construct Dataset based\ + \ on the batch prediction job.\n\n Args:\n bp_job: The batch prediction\ + \ job artifact.\n result: Tbe path to the file that contains Dataset\ + \ data.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ directory = bp_job.metadata['gcsOutputDirectory']\n data_source = {\n\ + \ 'tf_record_data_source': {\n 'file_patterns': [\n \ + \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ + \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ + \n" + image: python:3.7 + exec-write-bp-result-path-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _write_bp_result_path + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _write_bp_result_path(\n bp_job: dsl.Input[dsl.Artifact],\n\ + \ result: dsl.OutputPath('Dataset'),\n):\n \"\"\"Construct Dataset based\ + \ on the batch prediction job.\n\n Args:\n bp_job: The batch prediction\ + \ job artifact.\n result: Tbe path to the file that contains Dataset\ + \ data.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ directory = bp_job.metadata['gcsOutputDirectory']\n data_source = {\n\ + \ 'tf_record_data_source': {\n 'file_patterns': [\n \ + \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ + \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ + \n" + image: python:3.7 +pipelineInfo: + description: The AutoML Tabular pipeline. + name: automl-tabular-feature-selection-pipeline +root: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: exit-handler-1 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: exit-handler-1 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: exit-handler-1 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--additional_experiments: + componentInputParameter: additional_experiments + pipelinechannel--apply_feature_selection_tuning: + componentInputParameter: apply_feature_selection_tuning + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: cv_trainer_worker_pool_specs_override + pipelinechannel--data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + pipelinechannel--data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: distill_batch_predict_starting_replica_count + pipelinechannel--enable_probabilistic_inference: + componentInputParameter: enable_probabilistic_inference + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: fast_testing + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--model_display_name: + componentInputParameter: model_display_name + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--optimization_objective_precision_value: + componentInputParameter: optimization_objective_precision_value + pipelinechannel--optimization_objective_recall_value: + componentInputParameter: optimization_objective_recall_value + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--prediction_type: + componentInputParameter: prediction_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--quantiles: + componentInputParameter: quantiles + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_distillation: + componentInputParameter: run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: stage_2_num_selected_trials + pipelinechannel--stats_and_example_gen_dataflow_disk_size_gb: + componentInputParameter: stats_and_example_gen_dataflow_disk_size_gb + pipelinechannel--stats_and_example_gen_dataflow_machine_type: + componentInputParameter: stats_and_example_gen_dataflow_machine_type + pipelinechannel--stats_and_example_gen_dataflow_max_num_workers: + componentInputParameter: stats_and_example_gen_dataflow_max_num_workers + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: train_budget_milli_node_hours + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--transformations: + componentInputParameter: transformations + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + taskInfo: + name: exit-handler-1 + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Vertex model to upload this model as a version of. + isOptional: true + parameters: + additional_experiments: + description: Use this field to config private preview features. + isOptional: true + parameterType: STRUCT + apply_feature_selection_tuning: + defaultValue: false + description: tuning feature selection rate if true. + isOptional: true + parameterType: BOOLEAN + cv_trainer_worker_pool_specs_override: + description: 'The dictionary for overriding stage + + cv trainer worker pool spec.' + isOptional: true + parameterType: LIST + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + disable_early_stopping: + defaultValue: false + description: If disable easly stopping. + isOptional: true + parameterType: BOOLEAN + distill_batch_predict_machine_type: + defaultValue: n1-standard-16 + description: 'The prediction server machine type for + + batch predict component in the model distillation.' + isOptional: true + parameterType: STRING + distill_batch_predict_max_replica_count: + defaultValue: 25.0 + description: 'The max number of prediction server + + for batch predict component in the model distillation.' + isOptional: true + parameterType: NUMBER_INTEGER + distill_batch_predict_starting_replica_count: + defaultValue: 25.0 + description: 'The initial number of + + prediction server for batch predict component in the model distillation.' + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is enabled, the + + model will fit a distribution that captures the uncertainty of a + + prediction. At inference time, the predictive distribution is used to make + + a point prediction that minimizes the optimization objective. For example, + + the mean of a predictive distribution is the point prediction that + + minimizes RMSE loss. If quantiles are specified, then the quantiles of the + + distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluation_batch_explain_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch explain components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_max_replica_count: + defaultValue: 10.0 + description: 'The max number of prediction + + server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_explain_starting_replica_count: + defaultValue: 10.0 + description: 'The initial number of + + prediction server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + export_additional_model_without_custom_ops: + defaultValue: false + description: 'Whether to export additional + + model without custom TensorFlow operators.' + isOptional: true + parameterType: BOOLEAN + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: number of features to select for training. + isOptional: true + parameterType: NUMBER_INTEGER + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + optimization_objective: + description: 'For binary classification, "maximize-au-roc", + + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", + or + + "maximize-recall-at-precision". For multi class classification, + + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + + "minimize-rmsle".' + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when optimization_objective + + is ''maximize-recall-at-precision''. Must be between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when optimization_objective is + + ''maximize-precision-at-recall''. Must be between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to produce. + + "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + quantiles: + description: 'Quantiles to use for probabilistic inference. Up to 5 quantiles + + are allowed of values between 0 and 1, exclusive. Represents the quantiles + + to use for that objective. Quantiles must be unique.' + isOptional: true + parameterType: LIST + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + stage_1_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 1. + isOptional: true + parameterType: NUMBER_INTEGER + stage_1_tuner_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 1 tuner worker pool spec.' + isOptional: true + parameterType: LIST + stage_2_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stage_2_num_selected_trials: + defaultValue: 5.0 + description: Number of selected trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stats_and_example_gen_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in + + GB for stats_and_example_gen component.' + isOptional: true + parameterType: NUMBER_INTEGER + stats_and_example_gen_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for + + stats_and_example_gen component.' + isOptional: true + parameterType: STRING + stats_and_example_gen_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow + + workers for stats_and_example_gen component.' + isOptional: true + parameterType: NUMBER_INTEGER + stratified_split_key: + defaultValue: '' + description: The stratified_split column name. + isOptional: true + parameterType: STRING + study_spec_parameters_override: + description: The list for overriding study spec. + isOptional: true + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: float = The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transformations: + description: 'The path to a GCS file containing the transformations to + + apply.' + parameterType: STRING + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml new file mode 100644 index 0000000000..6b9892f263 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/automl_tabular_v2_pipeline.yaml @@ -0,0 +1,12701 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-v2 +# Description: The AutoML Tabular pipeline v2. +# Inputs: +# apply_feature_selection_tuning: bool [Default: False] +# bigquery_staging_full_dataset_id: str [Default: ''] +# cv_trainer_worker_pool_specs_override: list +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# disable_early_stopping: bool [Default: False] +# distill_batch_predict_machine_type: str [Default: 'n1-standard-16'] +# distill_batch_predict_max_replica_count: int [Default: 40.0] +# distill_batch_predict_starting_replica_count: int [Default: 25.0] +# enable_probabilistic_inference: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# evaluation_batch_explain_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_explain_max_replica_count: int [Default: 10.0] +# evaluation_batch_explain_starting_replica_count: int [Default: 10.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# export_additional_model_without_custom_ops: bool [Default: False] +# fast_testing: bool [Default: False] +# feature_selection_algorithm: str [Default: 'AMI'] +# feature_transform_engine_dataflow_disk_size_gb: int [Default: 40.0] +# feature_transform_engine_dataflow_machine_type: str [Default: 'n1-standard-16'] +# feature_transform_engine_dataflow_max_num_workers: int [Default: 25.0] +# legacy_transformations_path: str [Default: ''] +# location: str +# materialized_examples_format: str [Default: 'tfrecords_gzip'] +# max_selected_features: int [Default: 1000.0] +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# num_selected_features: int [Default: 0.0] +# optimization_objective: str +# optimization_objective_precision_value: float [Default: -1.0] +# optimization_objective_recall_value: float [Default: -1.0] +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# quantiles: list +# root_dir: str +# run_distillation: bool [Default: False] +# run_evaluation: bool [Default: False] +# run_feature_selection: bool [Default: False] +# stage_1_num_parallel_trials: int [Default: 35.0] +# stage_1_tuner_worker_pool_specs_override: list +# stage_1_tuning_result_artifact_uri: str [Default: ''] +# stage_2_num_parallel_trials: int [Default: 35.0] +# stage_2_num_selected_trials: int [Default: 5.0] +# stratified_split_key: str [Default: ''] +# study_spec_parameters_override: list +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# tf_custom_transformation_definitions: list +# tf_transform_execution_engine: str [Default: ''] +# tf_transformations_path: str [Default: ''] +# train_budget_milli_node_hours: float +# training_fraction: float [Default: -1.0] +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# Outputs: +# feature-attribution-2-feature_attributions: system.Metrics +# feature-attribution-3-feature_attributions: system.Metrics +# feature-attribution-feature_attributions: system.Metrics +# model-evaluation-2-evaluation_metrics: system.Metrics +# model-evaluation-3-evaluation_metrics: system.Metrics +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-cv-trainer: + executorLabel: exec-automl-tabular-cv-trainer + inputDefinitions: + artifacts: + materialized_cv_splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized cross-validation splits. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: AutoML Tabular tuning result. + parameters: + deadline_hours: + description: Number of hours the cross-validation trainer should run. + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-cv-trainer-2: + executorLabel: exec-automl-tabular-cv-trainer-2 + inputDefinitions: + artifacts: + materialized_cv_splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized cross-validation splits. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: AutoML Tabular tuning result. + parameters: + deadline_hours: + description: Number of hours the cross-validation trainer should run. + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble: + executorLabel: exec-automl-tabular-ensemble + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble-2: + executorLabel: exec-automl-tabular-ensemble-2 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble-3: + executorLabel: exec-automl-tabular-ensemble-3 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-infra-validator: + executorLabel: exec-automl-tabular-infra-validator + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-infra-validator-2: + executorLabel: exec-automl-tabular-infra-validator-2 + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-infra-validator-3: + executorLabel: exec-automl-tabular-infra-validator-3 + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-stage-1-tuner: + executorLabel: exec-automl-tabular-stage-1-tuner + inputDefinitions: + artifacts: + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + disable_early_stopping: + defaultValue: false + description: 'True if disable early stopping. Default + + value is false.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "model_type","categorical_value_spec": {"values": + + ["nn"]}}]' + isOptional: true + parameterType: LIST + tune_feature_selection_rate: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-stage-1-tuner-2: + executorLabel: exec-automl-tabular-stage-1-tuner-2 + inputDefinitions: + artifacts: + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + disable_early_stopping: + defaultValue: false + description: 'True if disable early stopping. Default + + value is false.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "model_type","categorical_value_spec": {"values": + + ["nn"]}}]' + isOptional: true + parameterType: LIST + tune_feature_selection_rate: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-bool-identity-2: + executorLabel: exec-bool-identity-2 + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-bool-identity-3: + executorLabel: exec-bool-identity-3 + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-calculate-training-parameters: + executorLabel: exec-calculate-training-parameters + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + run_distillation: + description: Whether to run distill in the training pipeline. + parameterType: BOOLEAN + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + reduce_search_space_mode: + parameterType: STRING + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_num_selected_trials: + parameterType: NUMBER_INTEGER + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-calculate-training-parameters-2: + executorLabel: exec-calculate-training-parameters-2 + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + run_distillation: + description: Whether to run distill in the training pipeline. + parameterType: BOOLEAN + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + reduce_search_space_mode: + parameterType: STRING + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_num_selected_trials: + parameterType: NUMBER_INTEGER + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-condition-2: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-3 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-3 + tasks: + automl-tabular-cv-trainer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-cv-trainer + dependentTasks: + - calculate-training-parameters + - importer + - training-configurator-and-validator + inputs: + artifacts: + materialized_cv_splits: + componentInputArtifact: pipelinechannel--merge-materialized-splits-splits + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + taskInfo: + name: automl-tabular-cv-trainer + automl-tabular-ensemble: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble + dependentTasks: + - automl-tabular-cv-trainer + - training-configurator-and-validator + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-cv-trainer + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble + automl-tabular-infra-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator + dependentTasks: + - automl-tabular-ensemble + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + taskInfo: + name: automl-tabular-infra-validator + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: check-if-is-eval + calculate-training-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: true + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters + condition-3: + componentRef: + name: comp-condition-3 + dependentTasks: + - automl-tabular-ensemble + - bool-identity + - model-upload + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble + pipelinechannel--automl-tabular-ensemble-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--automl-tabular-ensemble-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: importer + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-tabular-ensemble + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - calculate-training-parameters + inputs: + artifacts: + dataset_stats: + componentInputArtifact: pipelinechannel--feature-transform-engine-dataset_stats + instance_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + training_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-training_schema + parameters: + enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + optimization_objective_precision_value: + componentInputParameter: pipelinechannel--optimization_objective_precision_value + optimization_objective_recall_value: + componentInputParameter: pipelinechannel--optimization_objective_recall_value + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + quantiles: + componentInputParameter: pipelinechannel--quantiles + run_distill: + componentInputParameter: pipelinechannel--run_distillation + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + componentInputParameter: pipelinechannel--feature-transform-engine-split_example_counts + stage_1_deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters + stage_2_deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--merge-materialized-splits-splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--enable_probabilistic_inference: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-split_example_counts: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--optimization_objective_precision_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--optimization_objective_recall_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + feature-attribution: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution + dependentTasks: + - model-batch-explanation + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution + model-batch-explanation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - feature-attribution + - model-evaluation + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-4: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-5 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-7 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-5 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-7 + tasks: + automl-tabular-cv-trainer-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-cv-trainer-2 + dependentTasks: + - automl-tabular-stage-1-tuner + - calculate-training-parameters-2 + - training-configurator-and-validator-2 + inputs: + artifacts: + materialized_cv_splits: + componentInputArtifact: pipelinechannel--merge-materialized-splits-splits + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator-2 + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-stage-1-tuner + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters-2 + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_features: + componentInputParameter: pipelinechannel--num_selected_features + num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters-2 + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + taskInfo: + name: automl-tabular-cv-trainer-2 + automl-tabular-ensemble-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble-2 + dependentTasks: + - automl-tabular-cv-trainer-2 + - training-configurator-and-validator-2 + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator-2 + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator-2 + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-cv-trainer-2 + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble-2 + automl-tabular-infra-validator-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator-2 + dependentTasks: + - automl-tabular-ensemble-2 + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + taskInfo: + name: automl-tabular-infra-validator-2 + automl-tabular-stage-1-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-stage-1-tuner + dependentTasks: + - calculate-training-parameters-2 + - training-configurator-and-validator-2 + inputs: + artifacts: + feature_ranking: + componentInputArtifact: pipelinechannel--feature-transform-engine-feature_ranking + materialized_eval_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--split-materialized-data-materialized_train_split + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator-2 + transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_features: + componentInputParameter: pipelinechannel--num_selected_features + num_selected_trials: + taskOutputParameter: + outputParameterKey: stage_1_num_selected_trials + producerTask: calculate-training-parameters-2 + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + taskOutputParameter: + outputParameterKey: reduce_search_space_mode + producerTask: calculate-training-parameters-2 + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + tune_feature_selection_rate: + componentInputParameter: pipelinechannel--apply_feature_selection_tuning + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-tabular-stage-1-tuner + bool-identity-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity-2 + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: check-if-is-eval + bool-identity-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity-3 + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_distillation + taskInfo: + name: check-if-is-distillation + calculate-training-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters-2 + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: false + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters-2 + condition-5: + componentRef: + name: comp-condition-5 + dependentTasks: + - automl-tabular-ensemble-2 + - bool-identity-2 + - bool-identity-3 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-2 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-2 + pipelinechannel--bool-identity-2-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-2 + pipelinechannel--bool-identity-3-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-3 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--set-optional-inputs-model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: no-distill + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-3-Output'] + == 'false' + condition-7: + componentRef: + name: comp-condition-7 + dependentTasks: + - automl-tabular-ensemble-2 + - bool-identity-2 + - bool-identity-3 + - calculate-training-parameters-2 + - get-bigquery-destination-output-uri + - get-bigquery-destination-output-uri-2 + - training-configurator-and-validator-2 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + pipelinechannel--feature-transform-engine-instance_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + pipelinechannel--feature-transform-engine-transform_output: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + pipelinechannel--training-configurator-and-validator-2-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator-2 + pipelinechannel--training-configurator-and-validator-2-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator-2 + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + pipelinechannel--bool-identity-2-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-2 + pipelinechannel--bool-identity-3-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-3 + pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours: + taskOutputParameter: + outputParameterKey: distill_stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode: + taskOutputParameter: + outputParameterKey: reduce_search_space_mode + producerTask: calculate-training-parameters-2 + pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_train_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_train_split_uri + pipelinechannel--feature-transform-engine-bigquery_validation_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_validation_split_uri + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + pipelinechannel--feature_transform_engine_dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + pipelinechannel--get-bigquery-destination-output-uri-2-bigquery_destination_output_uri: + taskOutputParameter: + outputParameterKey: bigquery_destination_output_uri + producerTask: get-bigquery-destination-output-uri-2 + pipelinechannel--get-bigquery-destination-output-uri-bigquery_destination_output_uri: + taskOutputParameter: + outputParameterKey: bigquery_destination_output_uri + producerTask: get-bigquery-destination-output-uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: distill + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-3-Output'] + == 'true' + get-bigquery-destination-output-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-bigquery-destination-output-uri + inputs: + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_train_split_uri + model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + table_prefix: + runtimeValue: + constant: train + taskInfo: + name: get-bigquery-destination-output-uri + get-bigquery-destination-output-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-bigquery-destination-output-uri-2 + inputs: + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_validation_split_uri + model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + table_prefix: + runtimeValue: + constant: validation + taskInfo: + name: get-bigquery-destination-output-uri-2 + training-configurator-and-validator-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator-2 + dependentTasks: + - calculate-training-parameters-2 + inputs: + artifacts: + dataset_stats: + componentInputArtifact: pipelinechannel--feature-transform-engine-dataset_stats + instance_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + training_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-training_schema + parameters: + enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + optimization_objective_precision_value: + componentInputParameter: pipelinechannel--optimization_objective_precision_value + optimization_objective_recall_value: + componentInputParameter: pipelinechannel--optimization_objective_recall_value + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + quantiles: + componentInputParameter: pipelinechannel--quantiles + run_distill: + componentInputParameter: pipelinechannel--run_distillation + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + componentInputParameter: pipelinechannel--feature-transform-engine-split_example_counts + stage_1_deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + stage_2_deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters-2 + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator-2 + inputDefinitions: + artifacts: + pipelinechannel--feature-transform-engine-dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--merge-materialized-splits-splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--split-materialized-data-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--apply_feature_selection_tuning: + parameterType: BOOLEAN + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--enable_probabilistic_inference: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_train_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_validation_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-split_example_counts: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_transform_engine_dataflow_machine_type: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--num_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--optimization_objective_precision_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--optimization_objective_recall_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-5: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-6 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-6 + tasks: + condition-6: + componentRef: + name: comp-condition-6 + dependentTasks: + - model-upload-2 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + pipelinechannel--model-upload-2-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-2 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + pipelinechannel--bool-identity-2-Output: + componentInputParameter: pipelinechannel--bool-identity-2-Output + pipelinechannel--bool-identity-3-Output: + componentInputParameter: pipelinechannel--bool-identity-3-Output + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-2-Output'] + == 'true' + model-upload-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-6: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-2 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation-2 + tasks: + feature-attribution-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-2 + dependentTasks: + - model-batch-explanation-2 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-2 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-2 + model-batch-explanation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-2 + model-batch-predict-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-2 + model-evaluation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-2 + dependentTasks: + - model-batch-predict-2 + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-2 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation-2 + model-evaluation-import-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-2 + dependentTasks: + - feature-attribution-2 + - model-evaluation-2 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-2 + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-2 + model: + componentInputArtifact: pipelinechannel--model-upload-2-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-2-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-7: + dag: + outputs: + artifacts: + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-8 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-8 + tasks: + automl-tabular-ensemble-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble-3 + dependentTasks: + - automl-tabular-stage-1-tuner-2 + - distillation-stage-feature-transform-engine + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--feature-transform-engine-instance_schema + instance_baseline: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-2-instance_baseline + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-2-metadata + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: distillation-stage-feature-transform-engine + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-stage-1-tuner-2 + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble-3 + automl-tabular-infra-validator-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator-3 + dependentTasks: + - automl-tabular-ensemble-3 + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + taskInfo: + name: automl-tabular-infra-validator-3 + automl-tabular-stage-1-tuner-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-stage-1-tuner-2 + dependentTasks: + - distillation-stage-feature-transform-engine + - split-materialized-data-2 + inputs: + artifacts: + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data-2 + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data-2 + metadata: + componentInputArtifact: pipelinechannel--training-configurator-and-validator-2-metadata + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: distillation-stage-feature-transform-engine + parameters: + deadline_hours: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours + disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + runtimeValue: + constant: 1.0 + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_distillation: + runtimeValue: + constant: true + single_run_max_secs: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-tabular-stage-1-tuner-2 + condition-8: + componentRef: + name: comp-condition-8 + dependentTasks: + - automl-tabular-ensemble-3 + - model-upload-3 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-3 + pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + pipelinechannel--model-upload-3-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-3 + parameters: + pipelinechannel--automl-tabular-ensemble-3-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-3 + pipelinechannel--bool-identity-2-Output: + componentInputParameter: pipelinechannel--bool-identity-2-Output + pipelinechannel--bool-identity-3-Output: + componentInputParameter: pipelinechannel--bool-identity-3-Output + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-2-Output'] + == 'true' + distillation-stage-feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-distillation-stage-feature-transform-engine + dependentTasks: + - get-bp-bq-output-table + - get-bp-bq-output-table-2 + - get-transform-config-path + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + bigquery_train_full_table_uri: + taskOutputParameter: + outputParameterKey: bq_output_table_uri + producerTask: get-bp-bq-output-table + bigquery_validate_full_table_uri: + taskOutputParameter: + outputParameterKey: bq_output_table_uri + producerTask: get-bp-bq-output-table-2 + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_column: + componentInputParameter: pipelinechannel--target_column + transform_config_path: + taskOutputParameter: + outputParameterKey: transform_config_path + producerTask: get-transform-config-path + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: distillation-stage-feature-transform-engine + get-bp-bq-output-table: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-bp-bq-output-table + dependentTasks: + - model-batch-predict-3 + inputs: + artifacts: + bp_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-3 + taskInfo: + name: get-bp-bq-output-table + get-bp-bq-output-table-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-bp-bq-output-table-2 + dependentTasks: + - model-batch-predict-4 + inputs: + artifacts: + bp_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-4 + taskInfo: + name: get-bp-bq-output-table-2 + get-transform-config-path: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-transform-config-path + inputs: + artifacts: + transform_output_dir_artifact: + componentInputArtifact: pipelinechannel--feature-transform-engine-transform_output + taskInfo: + name: get-transform-config-path + model-batch-predict-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-3 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--get-bigquery-destination-output-uri-bigquery_destination_output_uri + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_train_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-3 + model-batch-predict-4: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-4 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + bigquery_destination_output_uri: + componentInputParameter: pipelinechannel--get-bigquery-destination-output-uri-2-bigquery_destination_output_uri + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_validation_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-4 + model-upload-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-3 + dependentTasks: + - automl-tabular-ensemble-3 + - automl-tabular-infra-validator-3 + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-3 + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + parameters: + display_name: + runtimeValue: + constant: automl-tabular-distill-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-3 + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-3 + split-materialized-data-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data-2 + dependentTasks: + - distillation-stage-feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: distillation-stage-feature-transform-engine + taskInfo: + name: split-materialized-data-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--feature-transform-engine-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-2-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--training-configurator-and-validator-2-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode: + parameterType: STRING + pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_train_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_validation_split_uri: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_transform_engine_dataflow_machine_type: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--get-bigquery-destination-output-uri-2-bigquery_destination_output_uri: + parameterType: STRING + pipelinechannel--get-bigquery-destination-output-uri-bigquery_destination_output_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-8: + dag: + outputs: + artifacts: + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-3 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation-3 + tasks: + feature-attribution-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-3 + dependentTasks: + - model-batch-explanation-3 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-3 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-3 + model-batch-explanation-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-3 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-3-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-3 + model-batch-predict-5: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-5 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-5 + model-evaluation-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-3 + dependentTasks: + - model-batch-predict-5 + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-5 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation-3 + model-evaluation-import-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-3 + dependentTasks: + - feature-attribution-3 + - model-evaluation-3 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-3 + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-3 + model: + componentInputArtifact: pipelinechannel--model-upload-3-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import-3 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-3-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-3-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + parameterType: STRING + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-distillation-stage-feature-transform-engine: + executorLabel: exec-distillation-stage-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + ''projectId.datasetId'' format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + ''vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}''. + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + bigquery_train_full_table_uri: + description: 'BigQuery full table id for our + + train split output by pre-distillation FTE with soft target included.' + parameterType: STRING + bigquery_validate_full_table_uri: + description: 'BigQuery full table id for our + + validation split output by pre-distillation FTE with soft target + + included.' + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + prediction_type: + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + target_column: + description: Target column of input data. + parameterType: STRING + transform_config_path: + description: 'Path to the transform config output by the + + pre-distillation FTE component.' + parameterType: STRING + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + gcp_resources: + description: 'GCP resources created by this component. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-exit-handler-1: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-4 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-4 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-2 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-4 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-4 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - feature-transform-engine + - merge-materialized-splits + - string-not-empty + inputs: + artifacts: + pipelinechannel--feature-transform-engine-dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--merge-materialized-splits-splits: + taskOutputArtifact: + outputArtifactKey: splits + producerTask: merge-materialized-splits + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + pipelinechannel--optimization_objective_precision_value: + componentInputParameter: pipelinechannel--optimization_objective_precision_value + pipelinechannel--optimization_objective_recall_value: + componentInputParameter: pipelinechannel--optimization_objective_recall_value + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_distillation: + componentInputParameter: pipelinechannel--run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--set-optional-inputs-model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + pipelinechannel--weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: stage_1_tuning_result_artifact_uri_not_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'true' + condition-4: + componentRef: + name: comp-condition-4 + dependentTasks: + - feature-transform-engine + - merge-materialized-splits + - split-materialized-data + - string-not-empty + inputs: + artifacts: + pipelinechannel--feature-transform-engine-dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-feature_ranking: + taskOutputArtifact: + outputArtifactKey: feature_ranking + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + pipelinechannel--merge-materialized-splits-splits: + taskOutputArtifact: + outputArtifactKey: splits + producerTask: merge-materialized-splits + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--split-materialized-data-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + pipelinechannel--split-materialized-data-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + parameters: + pipelinechannel--apply_feature_selection_tuning: + componentInputParameter: pipelinechannel--apply_feature_selection_tuning + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + pipelinechannel--enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--feature-transform-engine-bigquery_downsampled_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_downsampled_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_train_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_train_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-bigquery_validation_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_validation_split_uri + producerTask: feature-transform-engine + pipelinechannel--feature-transform-engine-split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + pipelinechannel--feature_transform_engine_dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--num_selected_features: + componentInputParameter: pipelinechannel--num_selected_features + pipelinechannel--optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + pipelinechannel--optimization_objective_precision_value: + componentInputParameter: pipelinechannel--optimization_objective_precision_value + pipelinechannel--optimization_objective_recall_value: + componentInputParameter: pipelinechannel--optimization_objective_recall_value + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--quantiles: + componentInputParameter: pipelinechannel--quantiles + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_distillation: + componentInputParameter: pipelinechannel--run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--set-optional-inputs-model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + pipelinechannel--weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: stage_1_tuning_result_artifact_uri_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'false' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--feature_transform_engine_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: pipelinechannel--dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: pipelinechannel--dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: pipelinechannel--feature_selection_algorithm + legacy_transformations_path: + componentInputParameter: pipelinechannel--legacy_transformations_path + location: + componentInputParameter: pipelinechannel--location + materialized_examples_format: + componentInputParameter: pipelinechannel--materialized_examples_format + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_feature_selection: + componentInputParameter: pipelinechannel--run_feature_selection + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--tf_auto_transform_features + tf_custom_transformation_definitions: + componentInputParameter: pipelinechannel--tf_custom_transformation_definitions + tf_transform_execution_engine: + componentInputParameter: pipelinechannel--tf_transform_execution_engine + tf_transformations_path: + componentInputParameter: pipelinechannel--tf_transformations_path + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + merge-materialized-splits: + cachingOptions: + enableCache: true + componentRef: + name: comp-merge-materialized-splits + dependentTasks: + - split-materialized-data + inputs: + artifacts: + split_0: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + split_1: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + taskInfo: + name: merge-materialized-splits + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + string-not-empty: + cachingOptions: + enableCache: true + componentRef: + name: comp-string-not-empty + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: check-if-is-stage-1-tuning-result-artifact-uri-not-empty + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--apply_feature_selection_tuning: + parameterType: BOOLEAN + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--dataset_level_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--dataset_level_transformations: + parameterType: LIST + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--enable_probabilistic_inference: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--feature_selection_algorithm: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_transform_engine_dataflow_machine_type: + parameterType: STRING + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--legacy_transformations_path: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--materialized_examples_format: + parameterType: STRING + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--num_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--optimization_objective_precision_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--optimization_objective_recall_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--run_feature_selection: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--tf_auto_transform_features: + parameterType: STRUCT + pipelinechannel--tf_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--tf_transform_execution_engine: + parameterType: STRING + pipelinechannel--tf_transformations_path: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-attribution: + executorLabel: exec-feature-attribution + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-2: + executorLabel: exec-feature-attribution-2 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-3: + executorLabel: exec-feature-attribution-3 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-get-bigquery-destination-output-uri: + executorLabel: exec-get-bigquery-destination-output-uri + inputDefinitions: + parameters: + bigquery_source_input_uri: + parameterType: STRING + model_display_name: + defaultValue: '' + isOptional: true + parameterType: STRING + table_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + bigquery_destination_output_uri: + parameterType: STRING + comp-get-bigquery-destination-output-uri-2: + executorLabel: exec-get-bigquery-destination-output-uri-2 + inputDefinitions: + parameters: + bigquery_source_input_uri: + parameterType: STRING + model_display_name: + defaultValue: '' + isOptional: true + parameterType: STRING + table_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + bigquery_destination_output_uri: + parameterType: STRING + comp-get-bp-bq-output-table: + executorLabel: exec-get-bp-bq-output-table + inputDefinitions: + artifacts: + bp_job: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The batch prediction job artifact. + outputDefinitions: + parameters: + bq_output_table_uri: + parameterType: STRING + comp-get-bp-bq-output-table-2: + executorLabel: exec-get-bp-bq-output-table-2 + inputDefinitions: + artifacts: + bp_job: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The batch prediction job artifact. + outputDefinitions: + parameters: + bq_output_table_uri: + parameterType: STRING + comp-get-transform-config-path: + executorLabel: exec-get-transform-config-path + inputDefinitions: + artifacts: + transform_output_dir_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Transform output dir str, output by FTE. + outputDefinitions: + parameters: + transform_config_path: + parameterType: STRING + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-merge-materialized-splits: + executorLabel: exec-merge-materialized-splits + inputDefinitions: + artifacts: + split_0: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The first materialized split. + split_1: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The second materialized split. + outputDefinitions: + artifacts: + splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-model-batch-explanation: + executorLabel: exec-model-batch-explanation + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-2: + executorLabel: exec-model-batch-explanation-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-3: + executorLabel: exec-model-batch-explanation-3 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-2: + executorLabel: exec-model-batch-predict-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-3: + executorLabel: exec-model-batch-predict-3 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-4: + executorLabel: exec-model-batch-predict-4 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-5: + executorLabel: exec-model-batch-predict-5 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-2: + executorLabel: exec-model-evaluation-2 + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-3: + executorLabel: exec-model-evaluation-3 + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-2: + executorLabel: exec-model-evaluation-import-2 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-3: + executorLabel: exec-model-evaluation-import-3 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-2: + executorLabel: exec-model-upload-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-3: + executorLabel: exec-model-upload-3 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-split-materialized-data-2: + executorLabel: exec-split-materialized-data-2 + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-string-not-empty: + executorLabel: exec-string-not-empty + inputDefinitions: + parameters: + value: + description: String value to be checked. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + comp-training-configurator-and-validator-2: + executorLabel: exec-training-configurator-and-validator-2 + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-automl-tabular-cv-trainer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-cv-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--valid_trials_completed_threshold=0.7\", \"--num_selected_trials=", + "{{$.inputs.parameters[''num_selected_trials'']}}", "\", \"--num_selected_features=", + "{{$.inputs.parameters[''num_selected_features'']}}", "\", \"--lro_job_info=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_cv_splits=", + "{{$.inputs.artifacts[''materialized_cv_splits''].uri}}", "\", \"--tuning_result_input_path=", + "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_custom_job=true\", + \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-cv-trainer-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-cv-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--valid_trials_completed_threshold=0.7\", \"--num_selected_trials=", + "{{$.inputs.parameters[''num_selected_trials'']}}", "\", \"--num_selected_features=", + "{{$.inputs.parameters[''num_selected_features'']}}", "\", \"--lro_job_info=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_cv_splits=", + "{{$.inputs.artifacts[''materialized_cv_splits''].uri}}", "\", \"--tuning_result_input_path=", + "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_custom_job=true\", + \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble-3: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-infra-validator: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-infra-validator-2: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-infra-validator-3: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-stage-1-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", + "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", + "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--num_selected_features=", "{{$.inputs.parameters[''num_selected_features'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=", + "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=", + "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=", + "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\", + \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-stage-1-tuner-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", + "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", + "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--num_selected_features=", "{{$.inputs.parameters[''num_selected_features'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=", + "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=", + "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=", + "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\", + \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-bool-identity-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-bool-identity-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-calculate-training-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n run_distillation: bool,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_num_selected_trials',\ + \ int),\n ('stage_1_single_run_max_secs', int),\n ('stage_2_deadline_hours',\ + \ float),\n ('stage_2_single_run_max_secs', int),\n ('distill_stage_1_deadline_hours',\ + \ float),\n ('reduce_search_space_mode', str),\n ],\n):\n \"\"\ + \"Calculates training parameters.\n\n Args:\n stage_1_num_parallel_trials:\ + \ Number of parallel trails for stage 1.\n train_budget_milli_node_hours:\ + \ The train budget of creating this model,\n expressed in milli node\ + \ hours i.e. 1,000 value in this field means 1 node\n hour.\n stage_2_num_parallel_trials:\ + \ Number of parallel trails for stage 2.\n run_distillation: Whether\ + \ to run distill in the training pipeline.\n is_skip_architecture_search:\ + \ If component is being called in the\n skip_architecture_search pipeline.\n\ + \ fast_testing: Internal flag used for presubmit tests.\n\n Returns:\n\ + \ stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\ + \ stage_1_num_selected_trials: Number of selected trails for stage\ + \ 1.\n stage_1_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 1\n training trial.\n stage_2_deadline_hours:\ + \ Maximum number of hours to run stage 2.\n stage_2_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 2\n training\ + \ trial.\n distill_stage_1_deadline_hours: Maximum number of hours\ + \ to run stage 1 for\n the model distillation.\n reduce_search_space_mode:\ + \ The reduce search space mode. Possible values:\n minimal, regular,\ + \ full.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ num_folds = 5\n distill_total_trials = 100\n\n stage_1_deadline_hours\ + \ = -1.0\n stage_1_num_selected_trials = -1\n stage_1_single_run_max_secs\ + \ = -1\n stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs =\ + \ -1\n distill_stage_1_deadline_hours = 1.0\n reduce_search_space_mode\ + \ = 'regular'\n\n if is_skip_architecture_search:\n stage_2_deadline_hours\ + \ = train_budget_milli_node_hours / 1000.0\n stage_2_single_run_max_secs\ + \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n else:\n hours = float(train_budget_milli_node_hours)\ + \ / 1000.0\n multiplier = stage_1_num_parallel_trials * hours / 500.0\n\ + \ stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\ + \ phase_2_rounds = int(\n math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\ + \ + 0.5\n )\n if phase_2_rounds < 1:\n phase_2_rounds = 1\n\n\ + \ # All of magic number \"1.3\" above is because the trial doesn't\n\ + \ # always finish in time_per_trial. 1.3 is an empirical safety margin\ + \ here.\n stage_1_deadline_secs = int(\n hours * 3600.0 - 1.3\ + \ * stage_1_single_run_max_secs * phase_2_rounds\n )\n\n if stage_1_deadline_secs\ + \ < hours * 3600.0 * 0.5:\n stage_1_deadline_secs = int(hours * 3600.0\ + \ * 0.5)\n # Phase 1 deadline is the same as phase 2 deadline in this\ + \ case. Phase 2\n # can't finish in time after the deadline is cut,\ + \ so adjust the time per\n # trial to meet the deadline.\n stage_1_single_run_max_secs\ + \ = int(\n stage_1_deadline_secs / (1.3 * phase_2_rounds)\n \ + \ )\n\n reduce_search_space_mode = 'minimal'\n if multiplier > 2:\n\ + \ reduce_search_space_mode = 'regular'\n if multiplier > 4:\n \ + \ reduce_search_space_mode = 'full'\n\n # Stage 2 number of trials\ + \ is stage_1_num_selected_trials *\n # num_folds, which should be equal\ + \ to phase_2_rounds *\n # stage_2_num_parallel_trials. Use this information\ + \ to calculate\n # stage_1_num_selected_trials:\n stage_1_num_selected_trials\ + \ = int(\n phase_2_rounds * stage_2_num_parallel_trials / num_folds\n\ + \ )\n stage_1_deadline_hours = stage_1_deadline_secs / 3600.0\n\n\ + \ stage_2_deadline_hours = hours - stage_1_deadline_hours\n stage_2_single_run_max_secs\ + \ = stage_1_single_run_max_secs\n\n if run_distillation:\n # All\ + \ of magic number \"1.3\" above is because the trial doesn't always\n \ + \ # finish in time_per_trial. 1.3 is an empirical safety margin here.\n\ + \ distill_stage_1_deadline_hours = (\n math.ceil(float(distill_total_trials)\ + \ / stage_1_num_parallel_trials)\n * stage_1_single_run_max_secs\n\ + \ * 1.3\n / 3600.0\n )\n\n if fast_testing:\n \ + \ distill_stage_1_deadline_hours = 0.2\n stage_1_deadline_hours = 0.2\n\ + \ stage_1_single_run_max_secs = 1\n stage_2_deadline_hours = 0.2\n\ + \ stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_num_selected_trials',\n 'stage_1_single_run_max_secs',\n\ + \ 'stage_2_deadline_hours',\n 'stage_2_single_run_max_secs',\n\ + \ 'distill_stage_1_deadline_hours',\n 'reduce_search_space_mode',\n\ + \ ],\n )(\n stage_1_deadline_hours,\n stage_1_num_selected_trials,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ + \ reduce_search_space_mode,\n )\n\n" + image: python:3.7 + exec-calculate-training-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n run_distillation: bool,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_num_selected_trials',\ + \ int),\n ('stage_1_single_run_max_secs', int),\n ('stage_2_deadline_hours',\ + \ float),\n ('stage_2_single_run_max_secs', int),\n ('distill_stage_1_deadline_hours',\ + \ float),\n ('reduce_search_space_mode', str),\n ],\n):\n \"\"\ + \"Calculates training parameters.\n\n Args:\n stage_1_num_parallel_trials:\ + \ Number of parallel trails for stage 1.\n train_budget_milli_node_hours:\ + \ The train budget of creating this model,\n expressed in milli node\ + \ hours i.e. 1,000 value in this field means 1 node\n hour.\n stage_2_num_parallel_trials:\ + \ Number of parallel trails for stage 2.\n run_distillation: Whether\ + \ to run distill in the training pipeline.\n is_skip_architecture_search:\ + \ If component is being called in the\n skip_architecture_search pipeline.\n\ + \ fast_testing: Internal flag used for presubmit tests.\n\n Returns:\n\ + \ stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\ + \ stage_1_num_selected_trials: Number of selected trails for stage\ + \ 1.\n stage_1_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 1\n training trial.\n stage_2_deadline_hours:\ + \ Maximum number of hours to run stage 2.\n stage_2_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 2\n training\ + \ trial.\n distill_stage_1_deadline_hours: Maximum number of hours\ + \ to run stage 1 for\n the model distillation.\n reduce_search_space_mode:\ + \ The reduce search space mode. Possible values:\n minimal, regular,\ + \ full.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ num_folds = 5\n distill_total_trials = 100\n\n stage_1_deadline_hours\ + \ = -1.0\n stage_1_num_selected_trials = -1\n stage_1_single_run_max_secs\ + \ = -1\n stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs =\ + \ -1\n distill_stage_1_deadline_hours = 1.0\n reduce_search_space_mode\ + \ = 'regular'\n\n if is_skip_architecture_search:\n stage_2_deadline_hours\ + \ = train_budget_milli_node_hours / 1000.0\n stage_2_single_run_max_secs\ + \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n else:\n hours = float(train_budget_milli_node_hours)\ + \ / 1000.0\n multiplier = stage_1_num_parallel_trials * hours / 500.0\n\ + \ stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\ + \ phase_2_rounds = int(\n math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\ + \ + 0.5\n )\n if phase_2_rounds < 1:\n phase_2_rounds = 1\n\n\ + \ # All of magic number \"1.3\" above is because the trial doesn't\n\ + \ # always finish in time_per_trial. 1.3 is an empirical safety margin\ + \ here.\n stage_1_deadline_secs = int(\n hours * 3600.0 - 1.3\ + \ * stage_1_single_run_max_secs * phase_2_rounds\n )\n\n if stage_1_deadline_secs\ + \ < hours * 3600.0 * 0.5:\n stage_1_deadline_secs = int(hours * 3600.0\ + \ * 0.5)\n # Phase 1 deadline is the same as phase 2 deadline in this\ + \ case. Phase 2\n # can't finish in time after the deadline is cut,\ + \ so adjust the time per\n # trial to meet the deadline.\n stage_1_single_run_max_secs\ + \ = int(\n stage_1_deadline_secs / (1.3 * phase_2_rounds)\n \ + \ )\n\n reduce_search_space_mode = 'minimal'\n if multiplier > 2:\n\ + \ reduce_search_space_mode = 'regular'\n if multiplier > 4:\n \ + \ reduce_search_space_mode = 'full'\n\n # Stage 2 number of trials\ + \ is stage_1_num_selected_trials *\n # num_folds, which should be equal\ + \ to phase_2_rounds *\n # stage_2_num_parallel_trials. Use this information\ + \ to calculate\n # stage_1_num_selected_trials:\n stage_1_num_selected_trials\ + \ = int(\n phase_2_rounds * stage_2_num_parallel_trials / num_folds\n\ + \ )\n stage_1_deadline_hours = stage_1_deadline_secs / 3600.0\n\n\ + \ stage_2_deadline_hours = hours - stage_1_deadline_hours\n stage_2_single_run_max_secs\ + \ = stage_1_single_run_max_secs\n\n if run_distillation:\n # All\ + \ of magic number \"1.3\" above is because the trial doesn't always\n \ + \ # finish in time_per_trial. 1.3 is an empirical safety margin here.\n\ + \ distill_stage_1_deadline_hours = (\n math.ceil(float(distill_total_trials)\ + \ / stage_1_num_parallel_trials)\n * stage_1_single_run_max_secs\n\ + \ * 1.3\n / 3600.0\n )\n\n if fast_testing:\n \ + \ distill_stage_1_deadline_hours = 0.2\n stage_1_deadline_hours = 0.2\n\ + \ stage_1_single_run_max_secs = 1\n stage_2_deadline_hours = 0.2\n\ + \ stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_num_selected_trials',\n 'stage_1_single_run_max_secs',\n\ + \ 'stage_2_deadline_hours',\n 'stage_2_single_run_max_secs',\n\ + \ 'distill_stage_1_deadline_hours',\n 'reduce_search_space_mode',\n\ + \ ],\n )(\n stage_1_deadline_hours,\n stage_1_num_selected_trials,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ + \ reduce_search_space_mode,\n )\n\n" + image: python:3.7 + exec-distillation-stage-feature-transform-engine: + container: + args: + - distillation_stage_feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--transform_config_path=", "{{$.inputs.parameters[''transform_config_path'']}}"]}' + - '{"Concat": ["--bigquery_train_full_table_uri=", "{{$.inputs.parameters[''bigquery_train_full_table_uri'']}}"]}' + - '{"Concat": ["--bigquery_validate_full_table_uri=", "{{$.inputs.parameters[''bigquery_validate_full_table_uri'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-feature-attribution: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-2: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-3: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-get-bigquery-destination-output-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_bigquery_destination_output_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_bigquery_destination_output_uri(\n bigquery_source_input_uri:\ + \ str,\n model_display_name: str = '',\n table_prefix: str = '',\n\ + ) -> NamedTuple('Outputs', [('bigquery_destination_output_uri', str),]):\n\ + \ \"\"\"Formats a bigquery output uri, including the model name and current\ + \ time in the table name.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n import datetime\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n bigquery_staging_dataset_uri = '.'.join(\n bigquery_source_input_uri.split('.')[:-1]\n\ + \ )\n curr_time = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%S-%f')\n\ + \ if model_display_name:\n model_display_name = f'{model_display_name}-'\n\ + \n if table_prefix:\n table_prefix = f'{table_prefix}-'\n\n return\ + \ collections.namedtuple(\n 'Outputs',\n [\n 'bigquery_destination_output_uri',\n\ + \ ],\n )(\n f'{bigquery_staging_dataset_uri}.{table_prefix}{model_display_name}{curr_time}',\n\ + \ )\n\n" + image: python:3.7 + exec-get-bigquery-destination-output-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_bigquery_destination_output_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_bigquery_destination_output_uri(\n bigquery_source_input_uri:\ + \ str,\n model_display_name: str = '',\n table_prefix: str = '',\n\ + ) -> NamedTuple('Outputs', [('bigquery_destination_output_uri', str),]):\n\ + \ \"\"\"Formats a bigquery output uri, including the model name and current\ + \ time in the table name.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n import datetime\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n bigquery_staging_dataset_uri = '.'.join(\n bigquery_source_input_uri.split('.')[:-1]\n\ + \ )\n curr_time = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%S-%f')\n\ + \ if model_display_name:\n model_display_name = f'{model_display_name}-'\n\ + \n if table_prefix:\n table_prefix = f'{table_prefix}-'\n\n return\ + \ collections.namedtuple(\n 'Outputs',\n [\n 'bigquery_destination_output_uri',\n\ + \ ],\n )(\n f'{bigquery_staging_dataset_uri}.{table_prefix}{model_display_name}{curr_time}',\n\ + \ )\n\n" + image: python:3.7 + exec-get-bp-bq-output-table: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_bp_bq_output_table + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_bp_bq_output_table(\n bp_job: dsl.Input[dsl.Artifact],\n\ + ) -> NamedTuple('Outputs', [('bq_output_table_uri', str),]):\n \"\"\"Gets\ + \ the output table uri from a batch prediction job.\n\n Args:\n bp_job:\ + \ The batch prediction job artifact.\n\n Returns:\n The uri of our output\ + \ table.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'bq_output_table_uri',\n ],\n )(\n f\"{bp_job.metadata['bigqueryOutputDataset']}.{bp_job.metadata['bigqueryOutputTable']}\"\ + ,\n )\n\n" + image: python:3.7 + exec-get-bp-bq-output-table-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_bp_bq_output_table + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_bp_bq_output_table(\n bp_job: dsl.Input[dsl.Artifact],\n\ + ) -> NamedTuple('Outputs', [('bq_output_table_uri', str),]):\n \"\"\"Gets\ + \ the output table uri from a batch prediction job.\n\n Args:\n bp_job:\ + \ The batch prediction job artifact.\n\n Returns:\n The uri of our output\ + \ table.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'bq_output_table_uri',\n ],\n )(\n f\"{bp_job.metadata['bigqueryOutputDataset']}.{bp_job.metadata['bigqueryOutputTable']}\"\ + ,\n )\n\n" + image: python:3.7 + exec-get-transform-config-path: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_transform_config_path + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_transform_config_path(\n transform_output_dir_artifact:\ + \ dsl.Input[dsl.Artifact],\n) -> NamedTuple('Outputs', [('transform_config_path',\ + \ str),]):\n \"\"\"Returns the path to the transform config file in the\ + \ transform_output_dir.\n\n Args:\n transform_output_dir_artifact: Transform\ + \ output dir str, output by FTE.\n\n Returns:\n The path of our transform\ + \ config.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n import os\n import tensorflow as tf\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(transform_output_dir_artifact.uri, 'r') as f:\n\ + \ transform_output_dir = f.read()\n\n transform_config_path = os.path.join(\n\ + \ transform_output_dir,\n 'feature_transform_engine',\n 'transform_config.json',\n\ + \ )\n\n return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'transform_config_path',\n ],\n )(\n transform_config_path,\n\ + \ )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-importer: + importer: + artifactUri: + runtimeParameter: uri + typeSchema: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + exec-merge-materialized-splits: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _merge_materialized_splits + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _merge_materialized_splits(\n split_0: dsl.InputPath('MaterializedSplit'),\n\ + \ split_1: dsl.InputPath('MaterializedSplit'),\n splits: dsl.OutputPath('MaterializedSplit'),\n\ + ):\n \"\"\"Merge two materialized splits.\n\n Args:\n split_0: The\ + \ first materialized split.\n split_1: The second materialized split.\n\ + \ splits: The merged materialized split.\n \"\"\"\n with open(split_0,\ + \ 'r') as f:\n split_0_content = f.read()\n with open(split_1, 'r')\ + \ as f:\n split_1_content = f.read()\n with open(splits, 'w') as f:\n\ + \ f.write(','.join([split_0_content, split_1_content]))\n\n" + image: python:3.7 + exec-model-batch-explanation: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-3: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-3: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-4: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-5: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-2: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-3: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-2: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-3: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-2: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-3: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-split-materialized-data-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-string-not-empty: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _string_not_empty + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _string_not_empty(value: str) -> str:\n \"\"\"Check if the input\ + \ string value is not empty.\n\n Args:\n value: String value to be checked.\n\ + \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ + \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ + \ \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + exec-training-configurator-and-validator-2: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: The AutoML Tabular pipeline v2. + name: automl-tabular-v2 +root: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: exit-handler-1 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: exit-handler-1 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: exit-handler-1 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--apply_feature_selection_tuning: + componentInputParameter: apply_feature_selection_tuning + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + pipelinechannel--dataset_level_transformations: + componentInputParameter: dataset_level_transformations + pipelinechannel--disable_early_stopping: + componentInputParameter: disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: distill_batch_predict_starting_replica_count + pipelinechannel--enable_probabilistic_inference: + componentInputParameter: enable_probabilistic_inference + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: fast_testing + pipelinechannel--feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + pipelinechannel--feature_transform_engine_dataflow_disk_size_gb: + componentInputParameter: feature_transform_engine_dataflow_disk_size_gb + pipelinechannel--feature_transform_engine_dataflow_machine_type: + componentInputParameter: feature_transform_engine_dataflow_machine_type + pipelinechannel--feature_transform_engine_dataflow_max_num_workers: + componentInputParameter: feature_transform_engine_dataflow_max_num_workers + pipelinechannel--legacy_transformations_path: + componentInputParameter: legacy_transformations_path + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--materialized_examples_format: + componentInputParameter: materialized_examples_format + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--num_selected_features: + componentInputParameter: num_selected_features + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--optimization_objective_precision_value: + componentInputParameter: optimization_objective_precision_value + pipelinechannel--optimization_objective_recall_value: + componentInputParameter: optimization_objective_recall_value + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--prediction_type: + componentInputParameter: prediction_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--quantiles: + componentInputParameter: quantiles + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_distillation: + componentInputParameter: run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--run_feature_selection: + componentInputParameter: run_feature_selection + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: stage_2_num_selected_trials + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + pipelinechannel--tf_custom_transformation_definitions: + componentInputParameter: tf_custom_transformation_definitions + pipelinechannel--tf_transform_execution_engine: + componentInputParameter: tf_transform_execution_engine + pipelinechannel--tf_transformations_path: + componentInputParameter: tf_transformations_path + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: train_budget_milli_node_hours + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Vertex Model to upload this model as a version to. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + apply_feature_selection_tuning: + defaultValue: false + description: tuning feature selection rate if true. + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'The BigQuery staging full dataset id for + + storing intermediate tables.' + isOptional: true + parameterType: STRING + cv_trainer_worker_pool_specs_override: + description: 'The dictionary for overriding stage + + cv trainer worker pool spec.' + isOptional: true + parameterType: LIST + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + description: 'Dataset-level custom + + transformation definitions in string format.' + isOptional: true + parameterType: LIST + dataset_level_transformations: + description: 'Dataset-level transformation configuration in + + string format.' + isOptional: true + parameterType: LIST + disable_early_stopping: + defaultValue: false + description: If disable easly stopping. + isOptional: true + parameterType: BOOLEAN + distill_batch_predict_machine_type: + defaultValue: n1-standard-16 + description: 'The prediction server machine type for + + batch predict component in the model distillation.' + isOptional: true + parameterType: STRING + distill_batch_predict_max_replica_count: + defaultValue: 40.0 + description: 'The max number of prediction server + + for batch predict component in the model distillation.' + isOptional: true + parameterType: NUMBER_INTEGER + distill_batch_predict_starting_replica_count: + defaultValue: 25.0 + description: 'The initial number of + + prediction server for batch predict component in the model distillation.' + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is enabled, the + + model will fit a distribution that captures the uncertainty of a + + prediction. At inference time, the predictive distribution is used to make + + a point prediction that minimizes the optimization objective. For example, + + the mean of a predictive distribution is the point prediction that + + minimizes RMSE loss. If quantiles are specified, then the quantiles of the + + distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluation_batch_explain_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch explain components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_max_replica_count: + defaultValue: 10.0 + description: 'The max number of prediction + + server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_explain_starting_replica_count: + defaultValue: 10.0 + description: 'The initial number of + + prediction server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + export_additional_model_without_custom_ops: + defaultValue: false + description: 'Whether to export additional + + model without custom TensorFlow operators.' + isOptional: true + parameterType: BOOLEAN + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + feature_selection_algorithm: + defaultValue: AMI + description: Feature selection algorithm. + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size + + in GB for feature transform engine component.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_transform_engine_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type + + for feature transform engine component.' + isOptional: true + parameterType: STRING + feature_transform_engine_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of + + Dataflow workers for feature transform engine component.' + isOptional: true + parameterType: NUMBER_INTEGER + legacy_transformations_path: + defaultValue: '' + description: Path to train spec transformations json. + isOptional: true + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: The format for the materialized examples. + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: Maximum number of features to select. + isOptional: true + parameterType: NUMBER_INTEGER + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features for feature selection, + + defaults to None, in which case all features are used.' + isOptional: true + parameterType: NUMBER_INTEGER + optimization_objective: + description: 'For binary classification, "maximize-au-roc", + + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", + or + + "maximize-recall-at-precision". For multi class classification, + + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + + "minimize-rmsle".' + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when optimization_objective + + is ''maximize-recall-at-precision''. Must be between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when optimization_objective is + + ''maximize-precision-at-recall''. Must be between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to produce. + + "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + quantiles: + description: 'Quantiles to use for probabilistic inference. Up to 5 quantiles + + are allowed of values between 0 and 1, exclusive. Represents the quantiles + + to use for that objective. Quantiles must be unique.' + isOptional: true + parameterType: LIST + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: Whether to apply feature selection or not. + isOptional: true + parameterType: BOOLEAN + stage_1_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 1. + isOptional: true + parameterType: NUMBER_INTEGER + stage_1_tuner_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 1 tuner worker pool spec.' + isOptional: true + parameterType: LIST + stage_1_tuning_result_artifact_uri: + defaultValue: '' + description: 'The stage 1 tuning result artifact GCS + + URI.' + isOptional: true + parameterType: STRING + stage_2_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stage_2_num_selected_trials: + defaultValue: 5.0 + description: Number of selected trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stratified_split_key: + defaultValue: '' + description: The stratified_split column name. + isOptional: true + parameterType: STRING + study_spec_parameters_override: + description: The list for overriding study spec. + isOptional: true + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: float = The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + description: 'List of auto transform features in the + + comma-separated string format.' + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + description: 'TF custom transformation definitions + + in string format.' + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: '' + description: 'Execution engine to run TF-based + + transformations. Currently supports "dataflow" or "bigquery"' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: Path to TF transformation configuration. + isOptional: true + parameterType: STRING + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_large_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_large_search_space.json new file mode 100644 index 0000000000..65e64d953d --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_large_search_space.json @@ -0,0 +1,158 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [50000, 70000, 90000, 110000, 130000, 150000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [4096, 8192, 16384, 32768, 65536] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.00007, + "max_value": 0.03 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [3, 5, 10] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [5, 10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 100, + "max_value": 700 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 3, + "max_value": 8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.05, + "max_value": 3.2 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 10000, + "max_value": 50000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.0000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 4, + "max_value": 10 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy", "focal_loss"] + } + }, + { + "parameter_id": "alpha_focal_loss", + "discrete_value_spec": { + "values": [0.1, 0.25, 0.5, 0.75, 0.9, 0.99] + } + }, + { + "parameter_id": "gamma_focal_loss", + "discrete_value_spec": { + "values": [0.0, 0.5, 1.0, 2.0, 3.0, 4.0] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false", "true"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_medium_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_medium_search_space.json new file mode 100644 index 0000000000..e7346ea9ae --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_medium_search_space.json @@ -0,0 +1,158 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [50000, 60000, 70000, 80000, 90000, 100000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [4096, 8192, 16384, 32768] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 0.03 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [5] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 200, + "max_value": 500 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 3, + "max_value": 7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.2, + "max_value": 2.5 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 10000, + "max_value": 50000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 4, + "max_value": 8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy", "focal_loss"] + } + }, + { + "parameter_id": "alpha_focal_loss", + "discrete_value_spec": { + "values": [0.1, 0.25, 0.5, 0.75, 0.9, 0.99] + } + }, + { + "parameter_id": "gamma_focal_loss", + "discrete_value_spec": { + "values": [0.0, 0.5, 1.0, 2.0, 3.0, 4.0] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_small_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_small_search_space.json new file mode 100644 index 0000000000..90ed01db8f --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_large_data_small_search_space.json @@ -0,0 +1,146 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [50000, 60000, 70000, 80000, 90000, 100000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [8192, 16384, 32768] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.0002, + "max_value": 0.02 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [5] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 100, + "max_value": 400 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 3, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.2, + "max_value": 2.5 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 10000, + "max_value": 50000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.3, + "max_value": 0.7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 10.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy"] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_large_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_large_search_space.json new file mode 100644 index 0000000000..b9350f33b6 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_large_search_space.json @@ -0,0 +1,158 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [50000, 60000, 70000, 80000, 90000, 100000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [1024, 2048, 4096, 8192, 16384] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.00007, + "max_value": 0.03 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [3, 5, 10] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [5, 10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 50, + "max_value": 500 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 2, + "max_value": 8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.05, + "max_value": 3.2 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 10000, + "max_value": 50000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.0000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 4, + "max_value": 10 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy", "focal_loss"] + } + }, + { + "parameter_id": "alpha_focal_loss", + "discrete_value_spec": { + "values": [0.1, 0.25, 0.5, 0.75, 0.9, 0.99] + } + }, + { + "parameter_id": "gamma_focal_loss", + "discrete_value_spec": { + "values": [0.0, 0.5, 1.0, 2.0, 3.0, 4.0] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false", "true"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_medium_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_medium_search_space.json new file mode 100644 index 0000000000..e7143fae84 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_medium_search_space.json @@ -0,0 +1,158 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [5000, 10000, 20000, 30000, 40000, 50000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [1024, 2048, 4096, 8192, 16384] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.00007, + "max_value": 0.02 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [5] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 50, + "max_value": 400 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.2, + "max_value": 2.5 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 10000, + "max_value": 50000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.0000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 4, + "max_value": 10 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy", "focal_loss"] + } + }, + { + "parameter_id": "alpha_focal_loss", + "discrete_value_spec": { + "values": [0.1, 0.25, 0.5, 0.75, 0.9, 0.99] + } + }, + { + "parameter_id": "gamma_focal_loss", + "discrete_value_spec": { + "values": [0.0, 0.5, 1.0, 2.0, 3.0, 4.0] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_small_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_small_search_space.json new file mode 100644 index 0000000000..46968c00c8 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_medium_data_small_search_space.json @@ -0,0 +1,146 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [10000, 20000, 30000, 40000, 50000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [1024, 4096, 8192, 16384] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 0.02 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [5] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 100, + "max_value": 300 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.3, + "max_value": 0.7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.2, + "max_value": 2.5 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 10000, + "max_value": 50000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.0000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.3, + "max_value": 0.7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy"] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_large_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_large_search_space.json new file mode 100644 index 0000000000..40d2e7f85b --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_large_search_space.json @@ -0,0 +1,158 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [10000, 20000, 30000, 40000, 50000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [512, 1024, 2048, 4096] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.00007, + "max_value": 0.03 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [3, 5, 10] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [5, 10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 50, + "max_value": 300 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 2, + "max_value": 7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.05, + "max_value": 3.2 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 1000, + "max_value": 10000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.0000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy", "focal_loss"] + } + }, + { + "parameter_id": "alpha_focal_loss", + "discrete_value_spec": { + "values": [0.1, 0.25, 0.5, 0.75, 0.9, 0.99] + } + }, + { + "parameter_id": "gamma_focal_loss", + "discrete_value_spec": { + "values": [0.0, 0.5, 1.0, 2.0, 3.0, 4.0] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false", "true"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_medium_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_medium_search_space.json new file mode 100644 index 0000000000..3a75145edf --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_medium_search_space.json @@ -0,0 +1,158 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [5000, 10000, 20000, 30000, 40000, 50000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [512, 1024, 2048, 4096] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 0.03 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [5] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 50, + "max_value": 200 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.2, + "max_value": 2.7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 1000, + "max_value": 10000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.0625, 0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.2, + "max_value": 0.8 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy", "focal_loss"] + } + }, + { + "parameter_id": "alpha_focal_loss", + "discrete_value_spec": { + "values": [0.1, 0.25, 0.5, 0.75, 0.9, 0.99] + } + }, + { + "parameter_id": "gamma_focal_loss", + "discrete_value_spec": { + "values": [0.0, 0.5, 1.0, 2.0, 3.0, 4.0] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_small_search_space.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_small_search_space.json new file mode 100644 index 0000000000..eb7a4c99f7 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/tabnet_params_small_data_small_search_space.json @@ -0,0 +1,146 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [5000, 10000, 15000, 20000, 25000, 30000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [512, 1024, 2048, 4096] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 0.02 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "large_category_dim", + "discrete_value_spec": { + "values": [5] + } + }, + { + "parameter_id": "large_category_thresh", + "discrete_value_spec": { + "values": [10] + } + }, + { + "parameter_id": "feature_dim", + "integer_value_spec": { + "min_value": 50, + "max_value": 200 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "feature_dim_ratio", + "double_value_spec": { + "min_value": 0.3, + "max_value": 0.7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_decision_steps", + "integer_value_spec": { + "min_value": 2, + "max_value": 6 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "relaxation_factor", + "double_value_spec": { + "min_value": 1.2, + "max_value": 2.5 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "decay_rate", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.999 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "decay_every", + "integer_value_spec": { + "min_value": 1000, + "max_value": 5000 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "sparsity_loss_weight", + "double_value_spec": { + "min_value": 0.000001, + "max_value": 0.001 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + { + "parameter_id": "batch_momentum", + "double_value_spec": { + "min_value": 0.5, + "max_value": 0.95 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "batch_size_ratio", + "discrete_value_spec": { + "values": [0.125, 0.25, 0.5] + } + }, + { + "parameter_id": "num_transformer_layers", + "integer_value_spec": { + "min_value": 2, + "max_value": 4 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "num_transformer_layers_ratio", + "double_value_spec": { + "min_value": 0.3, + "max_value": 0.7 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "class_weight", + "double_value_spec": { + "min_value": 1.0, + "max_value": 100.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "loss_function_type", + "categorical_value_spec": { + "values": ["weighted_cross_entropy"] + } + }, + { + "parameter_id": "yeo_johnson_transform", + "categorical_value_spec": { + "values": ["false"] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/wide_and_deep_params.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/wide_and_deep_params.json new file mode 100644 index 0000000000..6458b992d0 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/wide_and_deep_params.json @@ -0,0 +1,132 @@ +[ + { + "parameter_id": "max_steps", + "discrete_value_spec": { + "values": [5000, 10000, 20000, 30000, 40000, 50000] + } + }, + { + "parameter_id": "max_train_secs", + "discrete_value_spec": { + "values": [-1] + } + }, + { + "parameter_id": "learning_rate", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 0.0005 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "optimizer_type", + "categorical_value_spec": { + "values": ["adam", "ftrl", "sgd"] + } + }, + { + "parameter_id": "l1_regularization_strength", + "discrete_value_spec": { + "values": [0, 0.01, 0.02] + } + }, + { + "parameter_id": "l2_regularization_strength", + "discrete_value_spec": { + "values": [0, 0.01, 0.02] + } + }, + { + "parameter_id": "l2_shrinkage_regularization_strength", + "discrete_value_spec": { + "values": [0, 0.01, 0.02] + } + }, + { + "parameter_id": "beta_1", + "discrete_value_spec": { + "values": [0.7, 0.8, 0.9] + } + }, + { + "parameter_id": "beta_2", + "discrete_value_spec": { + "values": [0.8, 0.9, 0.999] + } + }, + { + "parameter_id": "hidden_units", + "categorical_value_spec": { + "values": ["30,30,30"] + } + }, + { + "parameter_id": "use_wide", + "categorical_value_spec": { + "values": ["true", "false"] + } + }, + { + "parameter_id": "embed_categories", + "categorical_value_spec": { + "values": ["true", "false"] + } + }, + { + "parameter_id": "dnn_dropout", + "discrete_value_spec": { + "values": [0, 0.1, 0.2] + } + }, + { + "parameter_id": "dnn_learning_rate", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 0.0005 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + { + "parameter_id": "dnn_optimizer_type", + "categorical_value_spec": { + "values": ["adam", "ftrl", "sgd"] + } + }, + { + "parameter_id": "dnn_l1_regularization_strength", + "discrete_value_spec": { + "values": [0, 0.01, 0.02] + } + }, + { + "parameter_id": "dnn_l2_regularization_strength", + "discrete_value_spec": { + "values": [0, 0.01, 0.02] + } + }, + { + "parameter_id": "dnn_l2_shrinkage_regularization_strength", + "discrete_value_spec": { + "values": [0, 0.01, 0.02] + } + }, + { + "parameter_id": "dnn_beta_1", + "discrete_value_spec": { + "values": [0.7, 0.8, 0.9] + } + }, + { + "parameter_id": "dnn_beta_2", + "discrete_value_spec": { + "values": [0.8, 0.9, 0.999] + } + }, + { + "parameter_id": "batch_size", + "discrete_value_spec": { + "values": [1024, 2048, 4096, 8192, 16384] + } + } +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/xgboost_params.json b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/xgboost_params.json new file mode 100644 index 0000000000..245a738beb --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/configs/xgboost_params.json @@ -0,0 +1,309 @@ +[{ + "parameter_id": "num_boost_round", + "discrete_value_spec": { + "values": [1, 5, 10, 15, 20] + } +}, { + "parameter_id": "early_stopping_rounds", + "discrete_value_spec": { + "values": [3, 5, 10] + } +}, { + "parameter_id": "base_score", + "discrete_value_spec": { + "values": [0.5] + } +}, { + "parameter_id": "booster", + "categorical_value_spec": { + "values": ["gbtree", "gblinear", "dart"] + }, + "conditional_parameter_specs": [{ + "parameter_spec": { + "parameter_id": "eta", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 1.0 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "gamma", + "discrete_value_spec": { + "values": [0, 10, 50, 100, 500, 1000] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "max_depth", + "integer_value_spec": { + "min_value": 6, + "max_value": 10 + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "min_child_weight", + "double_value_spec": { + "min_value": 0.0, + "max_value": 10.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "max_delta_step", + "discrete_value_spec": { + "values": [0.0, 1.0, 3.0, 5.0, 7.0, 9.0] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "subsample", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 1.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "colsample_bytree", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 1.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "colsample_bylevel", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 1.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "colsample_bynode", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 1.0 + }, + "scale_type": "UNIT_LINEAR_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "lambda", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 1.0 + }, + "scale_type": "UNIT_REVERSE_LOG_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart", "gblinear"] + } + }, { + "parameter_spec": { + "parameter_id": "alpha", + "double_value_spec": { + "min_value": 0.0001, + "max_value": 1.0 + }, + "scale_type": "UNIT_LOG_SCALE" + }, + "parent_categorical_values": { + "values": ["gbtree", "dart", "gblinear"] + } + }, { + "parameter_spec": { + "parameter_id": "tree_method", + "categorical_value_spec": { + "values": ["auto"] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "scale_pos_weight", + "discrete_value_spec": { + "values": [1.0] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "refresh_leaf", + "discrete_value_spec": { + "values": [1] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "process_type", + "categorical_value_spec": { + "values": ["default"] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "grow_policy", + "categorical_value_spec": { + "values": ["depthwise"] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "sampling_method", + "categorical_value_spec": { + "values": ["uniform"] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "sample_type", + "categorical_value_spec": { + "values": ["uniform"] + } + }, + "parent_categorical_values": { + "values": ["dart"] + } + }, { + "parameter_spec": { + "parameter_id": "normalize_type", + "categorical_value_spec": { + "values": ["tree"] + } + }, + "parent_categorical_values": { + "values": ["dart"] + } + }, { + "parameter_spec": { + "parameter_id": "rate_drop", + "discrete_value_spec": { + "values": [0.0] + } + }, + "parent_categorical_values": { + "values": ["dart"] + } + }, { + "parameter_spec": { + "parameter_id": "one_drop", + "discrete_value_spec": { + "values": [0] + } + }, + "parent_categorical_values": { + "values": ["dart"] + } + }, { + "parameter_spec": { + "parameter_id": "skip_drop", + "discrete_value_spec": { + "values": [0.0] + } + }, + "parent_categorical_values": { + "values": ["dart"] + } + }, { + "parameter_spec": { + "parameter_id": "num_parallel_tree", + "discrete_value_spec": { + "values": [1] + } + }, + "parent_categorical_values": { + "values": ["gblinear"] + } + }, { + "parameter_spec": { + "parameter_id": "feature_selector", + "categorical_value_spec": { + "values": ["cyclic"] + } + }, + "parent_categorical_values": { + "values": ["gblinear"] + } + }, { + "parameter_spec": { + "parameter_id": "top_k", + "discrete_value_spec": { + "values": [0] + } + }, + "parent_categorical_values": { + "values": ["gblinear"] + } + }, { + "parameter_spec": { + "parameter_id": "max_leaves", + "discrete_value_spec": { + "values": [0] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }, { + "parameter_spec": { + "parameter_id": "max_bin", + "discrete_value_spec": { + "values": [256] + } + }, + "parent_categorical_values": { + "values": ["gbtree", "dart"] + } + }] +}] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py new file mode 100644 index 0000000000..cb17bb7073 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/distillation_stage_feature_transform_engine.py @@ -0,0 +1,219 @@ +"""AutoML Feature Transform Engine component spec.""" + +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from kfp import dsl + + +@dsl.container_component +def distillation_stage_feature_transform_engine( + root_dir: str, + project: str, + location: str, + transform_config_path: str, + bigquery_train_full_table_uri: str, + bigquery_validate_full_table_uri: str, + target_column: str, + prediction_type: str, + materialized_data: dsl.Output[dsl.Dataset], + transform_output: dsl.Output[dsl.Artifact], + gcp_resources: dsl.OutputPath(str), + bigquery_staging_full_dataset_id: Optional[str] = '', + weight_column: Optional[str] = '', + dataflow_machine_type: Optional[str] = 'n1-standard-16', + dataflow_max_num_workers: Optional[int] = 25, + dataflow_disk_size_gb: Optional[int] = 40, + dataflow_subnetwork: Optional[str] = '', + dataflow_use_public_ips: Optional[bool] = True, + dataflow_service_account: Optional[str] = '', + encryption_spec_key_name: Optional[str] = '', + autodetect_csv_schema: Optional[bool] = False, +): + # fmt: off + """Feature Transform Engine (FTE) component to transform raw data to engineered features during model distilation. + + The FTE transform configuration is generated as part of the FTE stage prior + to distillation. This distillation-stage FTE component re-uses this config to + transform the input datasets with predicted outputs included (soft targets). + + Args: + root_dir: The Cloud Storage location to store the output. + project: Project to run feature transform engine. + location: Location for the created GCP services. + transform_config_path: Path to the transform config output by the pre-distillation FTE component. + bigquery_train_full_table_uri: BigQuery full table id for our train split output by pre-distillation FTE with soft target included. + bigquery_validate_full_table_uri: BigQuery full table id for our validation split output by pre-distillation FTE with soft target included. + target_column: Target column of input data. prediction_type (str): Model prediction type. One of "classification", "regression", "time_series". + bigquery_staging_full_dataset_id: Dataset in 'projectId.datasetId' format for storing intermediate-FTE BigQuery tables. If the specified dataset does not exist in BigQuery, FTE will create the dataset. If no bigquery_staging_full_dataset_id is specified, all intermediate tables will be stored in a dataset created under the provided project in the input data source's location during FTE execution called 'vertex_feature_transform_engine_staging_{location.replace('-', '_')}'. All tables generated by FTE will have a 30 day TTL. + weight_column: Weight column of input data. + dataflow_machine_type: The machine type used for dataflow jobs. If not set, default to n1-standard-16. + dataflow_max_num_workers: The number of workers to run the dataflow job. If not set, default to 25. + dataflow_disk_size_gb: The disk size, in gigabytes, to use on each Dataflow worker instance. If not set, default to 40. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications dataflow_use_public_ips (Optional[bool]): Specifies whether Dataflow workers use public IP addresses. + dataflow_service_account: Custom service account to run Dataflow jobs. + encryption_spec_key_name: Customer-managed encryption key. + + Returns: + materialized_data: The materialized dataset. + transform_output: The transform output artifact. + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + # fmt: on + + return dsl.ContainerSpec( + image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125', + command=[], + args=[ + 'distillation_stage_feature_transform_engine', + dsl.ConcatPlaceholder(items=['--project=', project]), + dsl.ConcatPlaceholder(items=['--location=', location]), + dsl.ConcatPlaceholder( + items=[ + '--transform_config_path=', + transform_config_path, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--bigquery_train_full_table_uri=', + bigquery_train_full_table_uri, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--bigquery_validate_full_table_uri=', + bigquery_validate_full_table_uri, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--bigquery_staging_full_dataset_id=', + bigquery_staging_full_dataset_id, + ] + ), + dsl.ConcatPlaceholder(items=['--target_column=', target_column]), + dsl.ConcatPlaceholder(items=['--prediction_type=', prediction_type]), + dsl.ConcatPlaceholder(items=['--weight_column=', weight_column]), + dsl.ConcatPlaceholder( + items=[ + '--error_file_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.txt', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--transform_output_artifact_path=', + transform_output.uri, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--transform_output_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/transform', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--materialized_examples_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/materialized', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--export_data_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/export', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--materialized_data_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/materialized_data', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--materialized_data_artifact_path=', + materialized_data.uri, + ] + ), + f'--job_name=feature-transform-engine-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}', + dsl.ConcatPlaceholder(items=['--dataflow_project=', project]), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_staging_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_staging', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_tmp_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_tmp', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_max_num_workers=', + dataflow_max_num_workers, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_machine_type=', + dataflow_machine_type, + ] + ), + '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125', + dsl.ConcatPlaceholder( + items=[ + '--dataflow_disk_size_gb=', + dataflow_disk_size_gb, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_subnetwork_fully_qualified=', + dataflow_subnetwork, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_use_public_ips=', + dataflow_use_public_ips, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_service_account=', + dataflow_service_account, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_kms_key=', + encryption_spec_key_name, + ] + ), + dsl.ConcatPlaceholder(items=['--gcp_resources_path=', gcp_resources]), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py new file mode 100644 index 0000000000..0bcd1a8915 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection.py @@ -0,0 +1,163 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Feature Ranking and Selection component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Dataset +from kfp.dsl import Input +from kfp.dsl import Output + + +# pylint: disable=dangerous-default-value,g-bare-generic,g-doc-args,unused-argument +@dsl.container_component +def tabular_feature_ranking_and_selection( + project: str, + location: str, + root_dir: str, + data_source: Input[Dataset], + target_column_name: str, + feature_ranking: Output[Artifact], + selected_features: Output[Artifact], + gcp_resources: dsl.OutputPath(str), + dataflow_machine_type: Optional[str] = 'n1-standard-16', + dataflow_max_num_workers: Optional[int] = 25, + dataflow_disk_size_gb: Optional[int] = 40, + dataflow_subnetwork: Optional[str] = '', + dataflow_use_public_ips: Optional[bool] = True, + dataflow_service_account: Optional[str] = '', + encryption_spec_key_name: Optional[str] = '', + algorithm: Optional[str] = 'AMI', + prediction_type: Optional[str] = 'unknown', + binary_classification: Optional[str] = 'false', + max_selected_features: Optional[int] = 1000, +): + # fmt: off + """Launches a feature selection task to pick top features. + + Args: + project: Project to run feature selection. + location: Location for running the feature selection. If not set, default to us-central1. + root_dir: The Cloud Storage location to store the output. + dataflow_machine_type: The machine type used for dataflow jobs. If not set, default to n1-standard-16. + dataflow_max_num_workers: The number of workers to run the dataflow job. If not set, default to 25. + dataflow_disk_size_gb: The disk size, in gigabytes, to use on each Dataflow worker instance. If not set, default to 40. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + dataflow_service_account: Custom service account to run dataflow jobs. + encryption_spec_key_name: Customer-managed encryption key. If this is set, then all resources will be encrypted with the provided encryption key. data_source(Dataset): The input dataset artifact which references csv, BigQuery, or TF Records. target_column_name(str): Target column name of the input dataset. + max_selected_features: number of features to select by the algorithm. If not set, default to 1000. + + Returns: + feature_ranking: the dictionary of feature names and feature ranking values. + selected_features: A json array of selected feature names. + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "tabular-feature-selection-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "args": ["feature_selection", "--data_source=', + data_source.uri, + '", "--target_column=', + target_column_name, + '", "--prediction_type=', + prediction_type, + '", "--binary_classification=', + binary_classification, + '", "--algorithm=', + algorithm, + '", "--feature_selection_dir=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/feature_selection/",' + f' "--job_name=tabular-feature-selection-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + '", "--dataflow_project=', + project, + '", "--error_file_path=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb",' + ' "--dataflow_staging_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_staging",' + ' "--dataflow_tmp_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_tmp",' + ' "--dataflow_max_num_workers=' + ), + dataflow_max_num_workers, + '", "--dataflow_worker_container_image=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125', + '", "--dataflow_machine_type=', + dataflow_machine_type, + '", "--dataflow_disk_size_gb=', + dataflow_disk_size_gb, + '", "--dataflow_subnetwork_fully_qualified=', + dataflow_subnetwork, + '", "--dataflow_use_public_ips=', + dataflow_use_public_ips, + '", "--dataflow_service_account=', + dataflow_service_account, + '", "--dataflow_kms_key=', + encryption_spec_key_name, + '", "--max_selected_features=', + max_selected_features, + '", "--feature_selection_result_path=', + feature_ranking.uri, + '", "--selected_features_path=', + selected_features.uri, + '", "--parse_json=true"]}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml new file mode 100644 index 0000000000..6934ea26e7 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_selection_pipeline.yaml @@ -0,0 +1,1555 @@ +# PIPELINE DEFINITION +# Name: feature-selection +# Description: Defines pipeline for feature transform engine component. +# Inputs: +# bigquery_staging_full_dataset_id: str [Default: ''] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_disk_size_gb: int [Default: 40.0] +# dataflow_machine_type: str [Default: 'n1-standard-16'] +# dataflow_max_num_workers: int [Default: 10.0] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# encryption_spec_key_name: str [Default: ''] +# feature_selection_algorithm: str [Default: 'AMI'] +# feature_selection_execution_engine: str [Default: 'bigquery'] +# location: str +# max_selected_features: int [Default: -1.0] +# optimization_objective: str +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# root_dir: str +# run_feature_selection: bool [Default: False] +# stage_1_deadline_hours: float [Default: -1.0] +# stage_2_deadline_hours: float [Default: -1.0] +# stratified_split_key: str [Default: ''] +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# training_fraction: float [Default: -1.0] +# validation_fraction: float [Default: -1.0] +# weight_column: str [Default: ''] +components: + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: Defines pipeline for feature transform engine component. + name: feature-selection +root: + dag: + tasks: + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: dataflow_service_account + dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + feature_selection_execution_engine: + componentInputParameter: feature_selection_execution_engine + forecasting_available_at_forecast_columns: + runtimeValue: + constant: [] + forecasting_context_window: + runtimeValue: + constant: -1.0 + forecasting_forecast_horizon: + runtimeValue: + constant: -1.0 + forecasting_holiday_regions: + runtimeValue: + constant: [] + forecasting_predefined_window_column: + runtimeValue: + constant: '' + forecasting_time_column: + runtimeValue: + constant: '' + forecasting_time_series_attribute_columns: + runtimeValue: + constant: [] + forecasting_time_series_identifier_columns: + runtimeValue: + constant: [] + forecasting_unavailable_at_forecast_columns: + runtimeValue: + constant: [] + forecasting_window_max_count: + runtimeValue: + constant: -1.0 + forecasting_window_stride_length: + runtimeValue: + constant: -1.0 + location: + componentInputParameter: location + materialized_examples_format: + runtimeValue: + constant: tfrecords_gzip + max_selected_features: + componentInputParameter: max_selected_features + multimodal_image_columns: + runtimeValue: + constant: [] + multimodal_tabular_columns: + runtimeValue: + constant: [] + multimodal_text_columns: + runtimeValue: + constant: [] + multimodal_timeseries_columns: + runtimeValue: + constant: [] + predefined_split_key: + componentInputParameter: predefined_split_key + prediction_type: + componentInputParameter: prediction_type + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + run_feature_selection: + componentInputParameter: run_feature_selection + stratified_split_key: + componentInputParameter: stratified_split_key + target_column: + componentInputParameter: target_column + test_fraction: + componentInputParameter: test_fraction + tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + tf_custom_transformation_definitions: + runtimeValue: + constant: [] + tf_transform_execution_engine: + runtimeValue: + constant: dataflow + tf_transformations_path: + runtimeValue: + constant: '' + training_fraction: + componentInputParameter: training_fraction + validation_fraction: + componentInputParameter: validation_fraction + weight_column: + componentInputParameter: weight_column + taskInfo: + name: feature-transform-engine + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + available_at_forecast_columns: + runtimeValue: + constant: [] + context_window: + runtimeValue: + constant: -1.0 + enable_probabilistic_inference: + runtimeValue: + constant: false + forecast_horizon: + runtimeValue: + constant: -1.0 + optimization_objective: + componentInputParameter: optimization_objective + optimization_objective_precision_value: + runtimeValue: + constant: -1.0 + optimization_objective_recall_value: + runtimeValue: + constant: -1.0 + prediction_type: + componentInputParameter: prediction_type + run_distill: + runtimeValue: + constant: false + run_evaluation: + runtimeValue: + constant: false + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + stage_1_deadline_hours: + componentInputParameter: stage_1_deadline_hours + stage_2_deadline_hours: + componentInputParameter: stage_2_deadline_hours + target_column: + componentInputParameter: target_column + time_column: + runtimeValue: + constant: '' + time_series_attribute_columns: + runtimeValue: + constant: [] + time_series_identifier_columns: + runtimeValue: + constant: [] + unavailable_at_forecast_columns: + runtimeValue: + constant: [] + weight_column: + componentInputParameter: weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + parameters: + bigquery_staging_full_dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + isOptional: true + parameterType: LIST + dataset_level_transformations: + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: bigquery + isOptional: true + parameterType: STRING + location: + parameterType: STRING + max_selected_features: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_INTEGER + optimization_objective: + parameterType: STRING + predefined_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + run_feature_selection: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + stage_1_deadline_hours: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + stratified_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + target_column: + parameterType: STRING + test_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + isOptional: true + parameterType: STRUCT + training_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py new file mode 100644 index 0000000000..6316f5de08 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/feature_transform_engine.py @@ -0,0 +1,698 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Feature Transform Engine component spec.""" + +from typing import Optional + +from kfp import dsl + + +@dsl.container_component +def feature_transform_engine( + root_dir: str, + project: str, + location: str, + dataset_stats: dsl.Output[dsl.Artifact], + materialized_data: dsl.Output[dsl.Dataset], + transform_output: dsl.Output[dsl.Artifact], + split_example_counts: dsl.OutputPath(str), + instance_schema: dsl.Output[dsl.Artifact], + training_schema: dsl.Output[dsl.Artifact], + bigquery_train_split_uri: dsl.OutputPath(str), + bigquery_validation_split_uri: dsl.OutputPath(str), + bigquery_test_split_uri: dsl.OutputPath(str), + bigquery_downsampled_test_split_uri: dsl.OutputPath(str), + feature_ranking: dsl.Output[dsl.Artifact], + gcp_resources: dsl.OutputPath(str), + dataset_level_custom_transformation_definitions: Optional[list] = [], + dataset_level_transformations: Optional[list] = [], + forecasting_time_column: Optional[str] = '', + forecasting_time_series_identifier_column: Optional[str] = None, + forecasting_time_series_identifier_columns: Optional[list] = [], + forecasting_time_series_attribute_columns: Optional[list] = [], + forecasting_unavailable_at_forecast_columns: Optional[list] = [], + forecasting_available_at_forecast_columns: Optional[list] = [], + forecasting_forecast_horizon: Optional[int] = -1, + forecasting_context_window: Optional[int] = -1, + forecasting_predefined_window_column: Optional[str] = '', + forecasting_window_stride_length: Optional[int] = -1, + forecasting_window_max_count: Optional[int] = -1, + forecasting_holiday_regions: Optional[list] = [], + forecasting_apply_windowing: Optional[bool] = True, + predefined_split_key: Optional[str] = '', + stratified_split_key: Optional[str] = '', + timestamp_split_key: Optional[str] = '', + training_fraction: Optional[float] = -1, + validation_fraction: Optional[float] = -1, + test_fraction: Optional[float] = -1, + stats_gen_execution_engine: Optional[str] = 'dataflow', + tf_transform_execution_engine: Optional[str] = 'dataflow', + tf_auto_transform_features: Optional[dict] = {}, + tf_custom_transformation_definitions: Optional[list] = [], + tf_transformations_path: Optional[str] = '', + legacy_transformations_path: Optional[str] = '', + target_column: Optional[str] = '', + weight_column: Optional[str] = '', + prediction_type: Optional[str] = '', + model_type: Optional[str] = None, + multimodal_tabular_columns: Optional[list] = [], + multimodal_timeseries_columns: Optional[list] = [], + multimodal_text_columns: Optional[list] = [], + multimodal_image_columns: Optional[list] = [], + run_distill: Optional[bool] = False, + run_feature_selection: Optional[bool] = False, + feature_selection_algorithm: Optional[str] = 'AMI', + feature_selection_execution_engine: Optional[str] = 'dataflow', + materialized_examples_format: Optional[str] = 'tfrecords_gzip', + max_selected_features: Optional[int] = 1000, + data_source_csv_filenames: Optional[str] = '', + data_source_bigquery_table_path: Optional[str] = '', + bigquery_staging_full_dataset_id: Optional[str] = '', + dataflow_machine_type: Optional[str] = 'n1-standard-16', + dataflow_max_num_workers: Optional[int] = 25, + dataflow_disk_size_gb: Optional[int] = 40, + dataflow_subnetwork: Optional[str] = '', + dataflow_use_public_ips: Optional[bool] = True, + dataflow_service_account: Optional[str] = '', + encryption_spec_key_name: Optional[str] = '', + autodetect_csv_schema: Optional[bool] = False, + group_columns: Optional[list] = None, + group_total_weight: float = 0.0, + temporal_total_weight: float = 0.0, + group_temporal_total_weight: float = 0.0, +): + # fmt: off + """Transforms raw data to engineered features. + + FTE performs dataset level transformations, data splitting, data statistic + generation, and TensorFlow-based row level transformations on the input + dataset based on the provided transformation configuration. + + Args: + root_dir: The Cloud Storage location to store the output. + project: Project to run feature transform engine. + location: Location for the created GCP services. + dataset_level_custom_transformation_definitions: List of dataset-level custom transformation definitions. Custom, bring-your-own dataset-level transform functions, where users can define and import their own transform function and use it with FTE's built-in transformations. Using custom transformations is an experimental feature and it is currently not supported during batch prediction. + + [ { "transformation": "ConcatCols", "module_path": "/path/to/custom_transform_fn_dlt.py", "function_name": "concat_cols" } ] Using custom transform function together with FTE's built-in transformations: .. code-block:: python [ { "transformation": "Join", "right_table_uri": "bq://test-project.dataset_test.table", "join_keys": [["join_key_col", "join_key_col"]] },{ "transformation": "ConcatCols", "cols": ["feature_1", "feature_2"], "output_col": "feature_1_2" } ] + + dataset_level_transformations: List of dataset-level transformations. + + [ { "transformation": "Join", "right_table_uri": "bq://test-project.dataset_test.table", "join_keys": [["join_key_col", "join_key_col"]] }, ... ] Additional information about FTE's currently supported built-in + transformations: + Join: Joins features from right_table_uri. For each join key, the left table keys will be included and the right table keys will be dropped. + Example: .. code-block:: python { "transformation": "Join", "right_table_uri": "bq://test-project.dataset_test.table", "join_keys": [["join_key_col", "join_key_col"]] } + Arguments: + right_table_uri: Right table BigQuery uri to join with input_full_table_id. + join_keys: Features to join on. For each nested list, the first element is a left table column and the second is its corresponding right table column. + TimeAggregate: Creates a new feature composed of values of an existing feature from a fixed time period ago or in the future. + Ex: A feature for sales by store 1 year ago. + Example: .. code-block:: python { "transformation": "TimeAggregate", "time_difference": 40, "time_difference_units": "DAY", "time_series_identifier_columns": ["store_id"], "time_column": "time_col", "time_difference_target_column": "target_col", "output_column": "output_col" } + Arguments: + time_difference: Number of time_difference_units to look back or into the future on our time_difference_target_column. + time_difference_units: Units of time_difference to look back or into the future on our time_difference_target_column. Must be one of * 'DAY' * 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER' * 'YEAR' + time_series_identifier_columns: Names of the time series identifier columns. + time_column: Name of the time column. + time_difference_target_column: Column we wish to get the value of time_difference time_difference_units in the past or future. + output_column: Name of our new time aggregate feature. + is_future: Whether we wish to look forward in time. Defaults to False. PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum: Performs a partition by reduce operation (one of max, min, avg, or sum) with a fixed historic time period. Ex: Getting avg sales (the reduce column) for each store (partition_by_column) over the previous 5 days (time_column, time_ago_units, and time_ago). + Example: .. code-block:: python { "transformation": "PartitionByMax", "reduce_column": "sell_price", "partition_by_columns": ["store_id", "state_id"], "time_column": "date", "time_ago": 1, "time_ago_units": "WEEK", "output_column": "partition_by_reduce_max_output" } + Arguments: + reduce_column: Column to apply the reduce operation on. Reduce operations include the + following: Max, Min, Avg, Sum. + partition_by_columns: List of columns to partition by. + time_column: Time column for the partition by operation's window function. + time_ago: Number of time_ago_units to look back on our target_column, starting from time_column (inclusive). + time_ago_units: Units of time_ago to look back on our target_column. Must be one of * 'DAY' * 'WEEK' + output_column: Name of our output feature. + + forecasting_time_column: Forecasting time column. + forecasting_time_series_identifier_column: [Deprecated] A forecasting time series identifier column. Raises an exception if used - use the "time_series_identifier_column" field instead. + forecasting_time_series_identifier_columns: The list of forecasting time series identifier columns. + forecasting_time_series_attribute_columns: Forecasting time series attribute columns. + forecasting_unavailable_at_forecast_columns: Forecasting unavailable at forecast columns. + forecasting_available_at_forecast_columns: Forecasting available at forecast columns. + forecasting_forecast_horizon: Forecasting horizon. + forecasting_context_window: Forecasting context window. + forecasting_predefined_window_column: Forecasting predefined window column. + forecasting_window_stride_length: Forecasting window stride length. + forecasting_window_max_count: Forecasting window max count. + forecasting_holiday_regions: The geographical region based on which the holiday effect is applied in modeling by adding holiday categorical array feature that include all holidays matching the date. This option only allowed when data granularity is day. By default, holiday effect modeling is disabled. To turn it on, specify the holiday region using this option. + Top level: * 'GLOBAL' + Second level: continental regions: * 'NA': North America + * 'JAPAC': Japan and Asia Pacific + * 'EMEA': Europe, the Middle East and Africa + * 'LAC': Latin America and the Caribbean + Third level: countries from ISO 3166-1 Country codes. + Valid regions: * 'GLOBAL' * 'NA' * 'JAPAC' * 'EMEA' * 'LAC' * 'AE' + * 'AR' * 'AT' * 'AU' * 'BE' * 'BR' * 'CA' * 'CH' * 'CL' * 'CN' * 'CO' + * 'CZ' * 'DE' * 'DK' * 'DZ' * 'EC' * 'EE' * 'EG' * 'ES' * 'FI' * 'FR' + * 'GB' * 'GR' * 'HK' * 'HU' * 'ID' * 'IE' * 'IL' * 'IN' * 'IR' * 'IT' + * 'JP' * 'KR' * 'LV' * 'MA' * 'MX' * 'MY' * 'NG' * 'NL' * 'NO' * 'NZ' + * 'PE' * 'PH' * 'PK' * 'PL' * 'PT' * 'RO' * 'RS' * 'RU' * 'SA' * 'SE' + * 'SG' * 'SI' * 'SK' * 'TH' * 'TR' * 'TW' * 'UA' * 'US' * 'VE' * 'VN' + * 'ZA' + forecasting_apply_windowing: Whether to apply window strategy. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + timestamp_split_key: Timestamp split key. + training_fraction: Fraction of input data for training. + validation_fraction: Fraction of input data for validation. + test_fraction: Fraction of input data for testing. + stats_gen_execution_engine: Execution engine to perform statistics generation. Can be one of: "dataflow" (by default) or "bigquery". Using "bigquery" as the execution engine is experimental. + tf_transform_execution_engine: Execution engine to perform row-level TF transformations. Can be one of: "dataflow" (by default) or "bigquery". Using "bigquery" as the execution engine is experimental and is for allowlisted customers only. In addition, executing on "bigquery" only supports auto transformations (i.e., specified by tf_auto_transform_features) and will raise an error when tf_custom_transformation_definitions or tf_transformations_path is set. + tf_auto_transform_features: Dict mapping auto and/or type-resolutions to TF transform features. FTE will automatically configure a set of built-in transformations for each feature based on its data statistics. If users do not want auto type resolution, but want the set of transformations for a given type to be automatically generated, they may specify pre-resolved transformations types. The following type hint dict keys are supported: * 'auto' * 'categorical' * 'numeric' * 'text' * 'timestamp' Example: `{ "auto": ["feature1"], "categorical": ["feature2", "feature3"], }`. Note that the target and weight column may not be included as an auto transformation unless users are running forecasting. + tf_custom_transformation_definitions: List of TensorFlow-based custom transformation definitions. Custom, bring-your-own transform functions, where users can define and import their own transform function and use it with FTE's built-in transformations. `[ { "transformation": "PlusOne", "module_path": "gs://bucket/custom_transform_fn.py", "function_name": "plus_one_transform" }, { "transformation": "MultiplyTwo", "module_path": "gs://bucket/custom_transform_fn.py", "function_name": "multiply_two_transform" } ] Using custom transform function together with FTE's built-in transformations: .. code-block:: python [ { "transformation": "CastToFloat", "input_columns": ["feature_1"], "output_columns": ["feature_1"] },{ "transformation": "PlusOne", "input_columns": ["feature_1"] "output_columns": ["feature_1_plused_one"] },{ "transformation": "MultiplyTwo", "input_columns": ["feature_1"] "output_columns": ["feature_1_multiplied_two"] } ] + tf_transformations_path: Path to TensorFlow-based transformation configuration. Path to a JSON file used to specified FTE's TF transformation configurations. In the following, we provide some sample transform configurations to demonstrate FTE's capabilities. All transformations on input columns are explicitly specified with FTE's built-in transformations. Chaining of multiple transformations on a single column is also supported. For example: .. code-block:: python [ { "transformation": "ZScale", "input_columns": ["feature_1"] }, { "transformation": "ZScale", "input_columns": ["feature_2"] } ]`. Additional information about FTE's currently supported built-in + transformations: + Datetime: Extracts datetime featues from a column containing timestamp strings. + Example: .. code-block:: python { "transformation": "Datetime", "input_columns": ["feature_1"], "time_format": "%Y-%m-%d" } + Arguments: + input_columns: A list with a single column to perform the datetime transformation on. + output_columns: Names of output columns, one for each datetime_features element. + time_format: Datetime format string. Time format is a combination of Date + Time Delimiter (optional) + Time (optional) directives. Valid date directives are as follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' # 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' # 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' # 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' # 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' # 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y' # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' # 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' # 11302018 * '%Y%m%d' # 20181130 Valid time delimiters are as follows * 'T' * ' ' Valid time directives are as follows * '%H:%M' # 23:59 * '%H:%M:%S' # + 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456] * '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 * '%H:%M:%S%z', # 23:59:58+0000 + datetime_features: List of datetime features to be extract. Each entry must be one of * 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK' * 'DAY_OF_YEAR' * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR' * 'MINUTE' * 'SECOND' Defaults to ['YEAR', 'MONTH', 'DAY', 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR'] + Log: Performs the natural log on a numeric column. + Example: .. code-block:: python { "transformation": "Log", "input_columns": ["feature_1"] } + Arguments: + input_columns: A list with a single column to perform the log transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + ZScale: Performs Z-scale normalization on a numeric column. + Example: .. code-block:: python { "transformation": "ZScale", "input_columns": ["feature_1"] } + Arguments: + input_columns: A list with a single column to perform the z-scale transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + Vocabulary: Converts strings to integers, where each unique string gets a unique integer representation. + Example: .. code-block:: python { "transformation": "Vocabulary", "input_columns": ["feature_1"] } + Arguments: + input_columns: A list with a single column to perform the vocabulary transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + top_k: Number of the most frequent words in the vocabulary to use for generating dictionary lookup indices. If not specified, all words in the vocabulary will be used. Defaults to None. + frequency_threshold: Limit the vocabulary only to words whose number of occurrences in the input exceeds frequency_threshold. If not specified, all words in the vocabulary will be included. If both top_k and frequency_threshold are specified, a word must satisfy both conditions to be included. Defaults to None. + Categorical: Transforms categorical columns to integer columns. + Example: .. code-block:: python { "transformation": "Categorical", "input_columns": ["feature_1"], "top_k": 10 } + Arguments: + input_columns: A list with a single column to perform the categorical transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + top_k: Number of the most frequent words in the vocabulary to use for generating dictionary lookup indices. If not specified, all words in the vocabulary will be used. + frequency_threshold: Limit the vocabulary only to words whose number of occurrences in the input exceeds frequency_threshold. If not specified, all words in the vocabulary will be included. If both top_k and frequency_threshold are specified, a word must satisfy both conditions to be included. + Reduce: Given a column where each entry is a numeric array, reduces arrays according to our reduce_mode. + Example: .. code-block:: python { "transformation": "Reduce", "input_columns": ["feature_1"], "reduce_mode": "MEAN", "output_columns": ["feature_1_mean"] } + Arguments: + input_columns: A list with a single column to perform the reduce transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + reduce_mode: One of * 'MAX' * 'MIN' * 'MEAN' * 'LAST_K' Defaults to 'MEAN'. + last_k: The number of last k elements when 'LAST_K' reduce mode is used. Defaults to 1. + SplitString: Given a column of strings, splits strings into token arrays. + Example: .. code-block:: python { "transformation": "SplitString", "input_columns": ["feature_1"], "separator": "$" } + Arguments: + input_columns: A list with a single column to perform the split string transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + separator: Separator to split input string into tokens. Defaults to ' '. + missing_token: Missing token to use when no string is included. Defaults to ' _MISSING_ '. + NGram: Given a column of strings, splits strings into token arrays where each token is an integer. + Example: .. code-block:: python { "transformation": "NGram", "input_columns": ["feature_1"], "min_ngram_size": 1, "max_ngram_size": 2, "separator": " " } + Arguments: + input_columns: A list with a single column to perform the n-gram transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + min_ngram_size: Minimum n-gram size. Must be a positive number and <= max_ngram_size. Defaults to 1. + max_ngram_size: Maximum n-gram size. Must be a positive number and >= min_ngram_size. Defaults to 2. + top_k: Number of the most frequent words in the vocabulary to use for generating dictionary lookup indices. If not specified, all words in the vocabulary will be used. Defaults to None. + frequency_threshold: Limit the dictionary's vocabulary only to words whose number of occurrences in the input exceeds frequency_threshold. If not specified, all words in the vocabulary will be included. If both top_k and frequency_threshold are specified, a word must satisfy both conditions to be included. Defaults to None. + separator: Separator to split input string into tokens. Defaults to ' '. + missing_token: Missing token to use when no string is included. Defaults to ' _MISSING_ '. + Clip: Given a numeric column, clips elements such that elements < min_value are assigned min_value, and elements > max_value are assigned max_value. + Example: .. code-block:: python { "transformation": "Clip", "input_columns": ["col1"], "output_columns": ["col1_clipped"], "min_value": 1., "max_value": 10., } + Arguments: + input_columns: A list with a single column to perform the n-gram transformation on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + min_value: Number where all values below min_value are set to min_value. If no min_value is provided, min clipping will not occur. Defaults to None. + max_value: Number where all values above max_value are set to max_value If no max_value is provided, max clipping will not occur. Defaults to None. + MultiHotEncoding: Performs multi-hot encoding on a categorical array column. + Example: .. code-block:: python { "transformation": "MultiHotEncoding", "input_columns": ["col1"], } The number of classes is determened by the largest number included in the input if it is numeric or the total number of unique values of the input if it is type str. If the input is has type str and an element contians separator tokens, the input will be split at separator indices, and the each element of the split list will be considered a seperate class. For example, + Input: .. code-block:: python [ ["foo bar"], # Example 0 ["foo", "bar"], # Example 1 ["foo"], # Example 2 ["bar"], # Example 3 ] Output (with default separator=" "): .. code-block:: python [ [1, 1], # Example 0 [1, 1], # Example 1 [1, 0], # Example 2 [0, 1], # Example 3 ] + Arguments: + input_columns: A list with a single column to perform the multi-hot-encoding on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + top_k: Number of the most frequent words in the vocabulary to use for generating dictionary lookup indices. If not specified, all words in the vocabulary will be used. Defaults to None. + frequency_threshold: Limit the dictionary's vocabulary only to words whose number of occurrences in the input exceeds frequency_threshold. If not specified, all words in the vocabulary will be included. If both top_k and frequency_threshold are specified, a word must satisfy both conditions to be included. Defaults to None. + separator: Separator to split input string into tokens. Defaults to ' '. + MaxAbsScale: Performs maximum absolute scaling on a numeric column. + Example: .. code-block:: python { "transformation": "MaxAbsScale", "input_columns": ["col1"], "output_columns": ["col1_max_abs_scaled"] } + Arguments: + input_columns: A list with a single column to perform max-abs-scale on. + output_columns: A list with a single output column name, corresponding to the output of our transformation. + Custom: Transformations defined in tf_custom_transformation_definitions are included here in the TensorFlow-based transformation configuration. For example, given the following tf_custom_transformation_definitions: .. code-block:: python [ { "transformation": "PlusX", "module_path": "gs://bucket/custom_transform_fn.py", "function_name": "plus_one_transform" } ] We can include the following transformation: .. code-block:: python { "transformation": "PlusX", "input_columns": ["col1"], "output_columns": ["col1_max_abs_scaled"] "x": 5 } Note that input_columns must still be included in our arguments and output_columns is optional. All other arguments are those defined in custom_transform_fn.py, which includes `"x"` in this case. See tf_custom_transformation_definitions above. legacy_transformations_path (Optional[str]) Deprecated. Prefer tf_auto_transform_features. Path to a GCS file containing JSON string for legacy style transformations. Note that legacy_transformations_path and tf_auto_transform_features cannot both be specified. + target_column: Target column of input data. + weight_column: Weight column of input data. + prediction_type: Model prediction type. One of "classification", "regression", "time_series". + run_distill: (deprecated) Whether the distillation should be applied to the training. + run_feature_selection: Whether the feature selection should be applied to the dataset. + feature_selection_algorithm: The algorithm of feature selection. One of "AMI", "CMIM", "JMIM", "MRMR", default to be "AMI". The algorithms available are: AMI(Adjusted Mutual Information): + Reference: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html Arrays are not yet supported in this algorithm. CMIM(Conditional Mutual Information Maximization): Reference paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, “Feature selection using Joint Mutual Information Maximisation,” Expert Systems with Applications, vol. 42, issue 22, 1 December 2015, Pages 8520-8532. JMIM(Joint Mutual Information Maximization + Reference: + paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, “Feature selection using Joint Mutual Information Maximisation,” Expert Systems with Applications, vol. 42, issue 22, 1 December 2015, Pages 8520-8532. MRMR(MIQ Minimum-redundancy Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long, and Chris Ding. "Feature selection based on mutual information criteria of max-dependency, max-relevance, and min-redundancy." IEEE Transactions on pattern analysis and machine intelligence 27, no. + 8: 1226-1238. + feature_selection_execution_engine: Execution engine to run feature selection, value can be dataflow, bigquery. + materialized_examples_format: The format to use for the materialized examples. Should be either 'tfrecords_gzip' (default) or 'parquet'. + max_selected_features: Maximum number of features to select. If specified, the transform config will be purged by only using the selected features that ranked top in the feature ranking, which has the ranking value for all supported features. If the number of input features is smaller than max_selected_features specified, we will still run the feature selection process and generate the feature ranking, no features will be excluded. The value will be set to 1000 by default if run_feature_selection is enabled. + data_source_csv_filenames: CSV input data source to run feature transform on. + data_source_bigquery_table_path: BigQuery input data source to run feature transform on. + bigquery_staging_full_dataset_id: Dataset in "projectId.datasetId" format for storing intermediate-FTE BigQuery tables. If the specified dataset does not exist in BigQuery, FTE will create the dataset. If no bigquery_staging_full_dataset_id is specified, all intermediate tables will be stored in a dataset created under the provided project in the input data source's location during FTE execution called "vertex_feature_transform_engine_staging_{location.replace('-', '_')}". All tables generated by FTE will have a 30 day TTL. + model_type: Model type, which we wish to engineer features for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, or tide. Defaults to the empty value, `None`. + multimodal_tabular_columns: List of multimodal tabular columns. Defaults to an empty list + multimodal_timeseries_columns: List of multimodal timeseries columns. Defaults to an empty list + multimodal_text_columns: List of multimodal text columns. Defaults to an empty list + multimodal_image_columns: List of multimodal image columns. Defaults to an empty list. + dataflow_machine_type: The machine type used for dataflow jobs. If not set, default to n1-standard-16. + dataflow_max_num_workers: The number of workers to run the dataflow job. If not set, default to 25. + dataflow_disk_size_gb: The disk size, in gigabytes, to use on each Dataflow worker instance. If not set, default to 40. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + dataflow_service_account: Custom service account to run Dataflow jobs. + encryption_spec_key_name: Customer-managed encryption key. + autodetect_csv_schema: If True, infers the column types + when importing CSVs into BigQuery. + + Returns: + dataset_stats: The stats of the dataset. + materialized_data: The materialized dataset. + transform_output: The transform output artifact. + split_example_counts: JSON string of data split example counts for train, validate, and test splits. + bigquery_train_split_uri: BigQuery URI for the train split to pass to the batch prediction component during distillation. + bigquery_validation_split_uri: BigQuery URI for the validation split to pass to the batch prediction component during distillation. + bigquery_test_split_uri: BigQuery URI for the test split to pass to the batch prediction component during evaluation. + bigquery_downsampled_test_split_uri: BigQuery URI for the downsampled test split to pass to the batch prediction component during batch explain. + instance_schema_path: Schema of input data to the tf_model at serving time. + training_schema_path: Schema of input data to the tf_model at training time. + feature_ranking: The ranking of features, all features supported in the dataset will be included. For "AMI" algorithm, array features won't be available in the ranking as arrays are not supported yet. + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + group_columns: A list of time series attribute column names that define the time series hierarchy. + group_total_weight: The weight of the loss for predictions aggregated over time series in the same group. + temporal_total_weight: The weight of the loss for predictions aggregated over the horizon for a single time series. + group_temporal_total_weight: The weight of the loss for predictions aggregated over both the horizon and time series in the same hierarchy group. + """ + # fmt: on + + return dsl.ContainerSpec( + image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125', + command=[], + args=[ + 'feature_transform_engine', + dsl.ConcatPlaceholder(items=['--project=', project]), + dsl.ConcatPlaceholder(items=['--location=', location]), + dsl.ConcatPlaceholder( + items=[ + '--dataset_level_custom_transformation_definitions=', + dataset_level_custom_transformation_definitions, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataset_level_transformations=', + dataset_level_transformations, + ] + ), + dsl.ConcatPlaceholder( + items=['--forecasting_time_column=', forecasting_time_column] + ), + dsl.IfPresentPlaceholder( + # Singular time series ID backwards support. + input_name='forecasting_time_series_identifier_column', + then=dsl.ConcatPlaceholder( + items=[ + '--forecasting_time_series_identifier_column=', + forecasting_time_series_identifier_column, + ] + ), + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_time_series_identifier_columns=', + forecasting_time_series_identifier_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_time_series_attribute_columns=', + forecasting_time_series_attribute_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_unavailable_at_forecast_columns=', + forecasting_unavailable_at_forecast_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_available_at_forecast_columns=', + forecasting_available_at_forecast_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_forecast_horizon=', + forecasting_forecast_horizon, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_context_window=', + forecasting_context_window, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_predefined_window_column=', + forecasting_predefined_window_column, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_window_stride_length=', + forecasting_window_stride_length, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_window_max_count=', + forecasting_window_max_count, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_holiday_regions=', + forecasting_holiday_regions, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_apply_windowing=', + forecasting_apply_windowing, + ] + ), + dsl.ConcatPlaceholder( + items=['--predefined_split_key=', predefined_split_key] + ), + dsl.ConcatPlaceholder( + items=['--stratified_split_key=', stratified_split_key] + ), + dsl.ConcatPlaceholder( + items=['--timestamp_split_key=', timestamp_split_key] + ), + dsl.ConcatPlaceholder( + items=['--training_fraction=', training_fraction] + ), + dsl.ConcatPlaceholder( + items=['--validation_fraction=', validation_fraction] + ), + dsl.ConcatPlaceholder(items=['--test_fraction=', test_fraction]), + dsl.ConcatPlaceholder( + items=[ + '--stats_gen_execution_engine=', + stats_gen_execution_engine, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--tf_transform_execution_engine=', + tf_transform_execution_engine, + ] + ), + dsl.IfPresentPlaceholder( + input_name='tf_auto_transform_features', + then=dsl.ConcatPlaceholder( + items=[ + '--tf_auto_transform_features=', + tf_auto_transform_features, + ] + ), + ), + dsl.ConcatPlaceholder( + items=[ + '--tf_custom_transformation_definitions=', + tf_custom_transformation_definitions, + ] + ), + dsl.ConcatPlaceholder( + items=['--tf_transformations_path=', tf_transformations_path] + ), + dsl.ConcatPlaceholder( + items=[ + '--legacy_transformations_path=', + legacy_transformations_path, + ] + ), + dsl.ConcatPlaceholder( + items=['--data_source_csv_filenames=', data_source_csv_filenames] + ), + dsl.ConcatPlaceholder( + items=[ + '--data_source_bigquery_table_path=', + data_source_bigquery_table_path, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--bigquery_staging_full_dataset_id=', + bigquery_staging_full_dataset_id, + ] + ), + dsl.ConcatPlaceholder(items=['--target_column=', target_column]), + dsl.ConcatPlaceholder(items=['--weight_column=', weight_column]), + dsl.ConcatPlaceholder(items=['--prediction_type=', prediction_type]), + dsl.IfPresentPlaceholder( + input_name='model_type', + then=dsl.ConcatPlaceholder(items=['--model_type=', model_type]), + ), + dsl.ConcatPlaceholder( + items=[ + '--multimodal_tabular_columns=', + multimodal_tabular_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--multimodal_timeseries_columns=', + multimodal_timeseries_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--multimodal_text_columns=', + multimodal_text_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--multimodal_image_columns=', + multimodal_image_columns, + ] + ), + dsl.ConcatPlaceholder(items=['--run_distill=', run_distill]), + dsl.ConcatPlaceholder( + items=['--run_feature_selection=', run_feature_selection] + ), + dsl.ConcatPlaceholder( + items=[ + '--materialized_examples_format=', + materialized_examples_format, + ] + ), + dsl.ConcatPlaceholder( + items=['--max_selected_features=', max_selected_features] + ), + dsl.ConcatPlaceholder( + items=[ + '--feature_selection_staging_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/feature_selection_staging_dir', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--feature_selection_algorithm=', + feature_selection_algorithm, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--feature_selection_execution_engine=', + feature_selection_execution_engine, + ] + ), + dsl.ConcatPlaceholder( + items=['--feature_ranking_path=', feature_ranking.uri] + ), + dsl.ConcatPlaceholder( + items=[ + '--error_file_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.txt', + ] + ), + dsl.ConcatPlaceholder( + items=['--stats_result_path=', dataset_stats.uri] + ), + dsl.ConcatPlaceholder( + items=['--transform_output_artifact_path=', transform_output.uri] + ), + dsl.ConcatPlaceholder( + items=[ + '--transform_output_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/transform', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--materialized_examples_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/materialized', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--export_data_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/export', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--materialized_data_path=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/materialized_data', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--materialized_data_artifact_path=', + materialized_data.uri, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--bigquery_train_split_uri_path=', + bigquery_train_split_uri, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--bigquery_validation_split_uri_path=', + bigquery_validation_split_uri, + ] + ), + dsl.ConcatPlaceholder( + items=['--bigquery_test_split_uri_path=', bigquery_test_split_uri] + ), + dsl.ConcatPlaceholder( + items=[ + '--bigquery_downsampled_test_split_uri_path=', + bigquery_downsampled_test_split_uri, + ] + ), + dsl.ConcatPlaceholder( + items=['--split_example_counts_path=', split_example_counts] + ), + dsl.ConcatPlaceholder( + items=['--instance_schema_path=', instance_schema.path] + ), + dsl.ConcatPlaceholder( + items=['--training_schema_path=', training_schema.path] + ), + f'--job_name=feature-transform-engine-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}', + dsl.ConcatPlaceholder(items=['--dataflow_project=', project]), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_staging_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_staging', + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_tmp_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_tmp', + ] + ), + dsl.ConcatPlaceholder( + items=['--dataflow_max_num_workers=', dataflow_max_num_workers] + ), + dsl.ConcatPlaceholder( + items=['--dataflow_machine_type=', dataflow_machine_type] + ), + '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125', + '--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125', + dsl.ConcatPlaceholder( + items=['--dataflow_disk_size_gb=', dataflow_disk_size_gb] + ), + dsl.ConcatPlaceholder( + items=[ + '--dataflow_subnetwork_fully_qualified=', + dataflow_subnetwork, + ] + ), + dsl.ConcatPlaceholder( + items=['--dataflow_use_public_ips=', dataflow_use_public_ips] + ), + dsl.ConcatPlaceholder( + items=['--dataflow_service_account=', dataflow_service_account] + ), + dsl.ConcatPlaceholder( + items=['--dataflow_kms_key=', encryption_spec_key_name] + ), + dsl.ConcatPlaceholder( + items=['--autodetect_csv_schema=', autodetect_csv_schema] + ), + dsl.ConcatPlaceholder(items=['--gcp_resources_path=', gcp_resources]), + dsl.IfPresentPlaceholder( + input_name='group_columns', + then=dsl.ConcatPlaceholder( + items=['--group_columns=', group_columns] + ), + ), + dsl.IfPresentPlaceholder( + input_name='group_total_weight', + then=dsl.ConcatPlaceholder( + items=['--group_total_weight=', group_total_weight] + ), + ), + dsl.IfPresentPlaceholder( + input_name='temporal_total_weight', + then=dsl.ConcatPlaceholder( + items=['--temporal_total_weight=', temporal_total_weight] + ), + ), + dsl.IfPresentPlaceholder( + input_name='group_temporal_total_weight', + then=dsl.ConcatPlaceholder( + items=[ + '--group_temporal_total_weight=', + group_temporal_total_weight, + ] + ), + ), + dsl.ConcatPlaceholder( + items=[ + '--encryption_spec_key_name=', + encryption_spec_key_name, + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py new file mode 100644 index 0000000000..fdcf4b357c --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job.py @@ -0,0 +1,213 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Tabnet Hyperparameter Tuning component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input + + +@dsl.container_component +def tabnet_hyperparameter_tuning_job( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + study_spec_metric_id: str, + study_spec_metric_goal: str, + study_spec_parameters_override: list, + max_trial_count: int, + parallel_trial_count: int, + instance_baseline: Input[Artifact], + metadata: Input[Artifact], + materialized_train_split: Input[Artifact], + materialized_eval_split: Input[Artifact], + transform_output: Input[Artifact], + training_schema_uri: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + instance_schema_uri: dsl.OutputPath(str), + prediction_schema_uri: dsl.OutputPath(str), + trials: dsl.OutputPath(str), + prediction_docker_uri_output: dsl.OutputPath(str), + execution_metrics: dsl.OutputPath(dict), + weight_column: Optional[str] = '', + enable_profiler: Optional[bool] = False, + cache_data: Optional[str] = 'auto', + seed: Optional[int] = 1, + eval_steps: Optional[int] = 0, + eval_frequency_secs: Optional[int] = 600, + max_failed_trial_count: Optional[int] = 0, + study_spec_algorithm: Optional[str] = 'ALGORITHM_UNSPECIFIED', + study_spec_measurement_selection_type: Optional[str] = 'BEST_MEASUREMENT', + training_machine_spec: Optional[dict] = {'machine_type': 'c2-standard-16'}, + training_disk_spec: Optional[dict] = { + 'boot_disk_type': 'pd-ssd', + 'boot_disk_size_gb': 100, + }, + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Tunes TabNet hyperparameters using Vertex HyperparameterTuningJob API. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + weight_column: The weight column name. + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or negative, it means run evaluation on the whole validation dataset. If set to 0, it means run evaluation for a fixed number of samples. + eval_frequency_secs: Frequency at which evaluation and checkpointing will take place. + study_spec_metric_id: Metric to optimize, possible values: [ 'loss', 'average_loss', 'rmse', 'mae', 'mql', 'accuracy', 'auc', 'precision', 'recall']. + study_spec_metric_goal: Optimization goal of the metric, possible values: "MAXIMIZE", "MINIMIZE". + study_spec_parameters_override: List of dictionaries representing parameters to optimize. The dictionary key is the parameter_id, which is passed to training job as a command line argument, and the dictionary value is the parameter specification of the metric. + max_trial_count: The desired total number of trials. + parallel_trial_count: The desired number of trials to run in parallel. + max_failed_trial_count: The number of failed trials that need to be seen before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides how many trials must fail before the whole job fails. + study_spec_algorithm: The search algorithm specified for the study. One of 'ALGORITHM_UNSPECIFIED', 'GRID_SEARCH', or 'RANDOM_SEARCH'. + study_spec_measurement_selection_type: Which measurement to use if/when the service automatically selects the final measurement from previously reported intermediate measurements. One of "BEST_MEASUREMENT" or "LAST_MEASUREMENT". + training_machine_spec: The training machine spec. See https://cloud.google.com/compute/docs/machine-types for options. + training_disk_spec: The training disk spec. + instance_baseline: The path to a JSON file for baseline values. + metadata: Amount of time in seconds to run the trainer for. + materialized_train_split: The path to the materialized train split. + materialized_eval_split: The path to the materialized validation split. + transform_output: The path to transform output. + training_schema_uri: The path to the training schema. + encryption_spec_key_name: The KMS key name. + + Returns: + gcp_resources: Serialized gcp_resources proto tracking the custom training job. + instance_schema_uri: The path to the instance schema. + prediction_schema_uri: The path to the prediction schema. + trials: The path to the hyperparameter tuning trials + prediction_docker_uri_output: The URI of the prediction container. + execution_metrics: Core metrics in dictionary of hyperparameter tuning job execution. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.hyperparameter_tuning_job.launcher', + ], + args=[ + '--type', + 'HyperparameterTuningJobWithMetrics', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--execution_metrics', + execution_metrics, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "tabnet-hyperparameter-tuning-job-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + '"}, "study_spec": {"metrics": [{"metric_id": "', + study_spec_metric_id, + '", "goal": "', + study_spec_metric_goal, + '"}], "parameters": ', + study_spec_parameters_override, + ', "algorithm": "', + study_spec_algorithm, + '", "measurement_selection_type": "', + study_spec_measurement_selection_type, + '"}, "max_trial_count": ', + max_trial_count, + ', "parallel_trial_count": ', + parallel_trial_count, + ', "max_failed_trial_count": ', + max_failed_trial_count, + ( + ', "trial_job_spec": {"worker_pool_specs":' + ' [{"replica_count":"' + ), + '1', + '", "machine_spec": ', + training_machine_spec, + ', "disk_spec": ', + training_disk_spec, + ', "container_spec": {"image_uri":"', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20231002_0125', + '", "args": ["--target_column=', + target_column, + '", "--weight_column=', + weight_column, + '", "--model_type=', + prediction_type, + '", "--prediction_docker_uri=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125', + '", "--prediction_docker_uri_artifact_path=', + prediction_docker_uri_output, + '", "--baseline_path=', + instance_baseline.uri, + '", "--metadata_path=', + metadata.uri, + '", "--transform_output_path=', + transform_output.uri, + '", "--training_schema_path=', + training_schema_uri.uri, + '", "--instance_schema_path=', + instance_schema_uri, + '", "--prediction_schema_path=', + prediction_schema_uri, + '", "--trials_path=', + trials, + '", "--job_dir=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train",' + ' "--training_data_path=' + ), + materialized_train_split.uri, + '", "--validation_data_path=', + materialized_eval_split.uri, + '", "--enable_profiler=', + enable_profiler, + '", "--cache_data=', + cache_data, + '", "--seed=', + seed, + '", "--measurement_selection_type=', + study_spec_measurement_selection_type, + '", "--metric_goal=', + study_spec_metric_goal, + '", "--eval_steps=', + eval_steps, + '", "--eval_frequency_secs=', + eval_frequency_secs, + '", "--generate_feature_importance=true"]}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml new file mode 100644 index 0000000000..afa6cf7f4c --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_hyperparameter_tuning_job_pipeline.yaml @@ -0,0 +1,4766 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-tabnet-hyperparameter-tuning-job +# Description: The TabNet built-in algorithm HyperparameterTuningJob pipeline. +# Inputs: +# bigquery_staging_full_dataset_id: str [Default: ''] +# cache_data: str [Default: 'auto'] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# enable_profiler: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# eval_frequency_secs: int [Default: 600.0] +# eval_steps: int [Default: 0.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# feature_selection_algorithm: str [Default: 'AMI'] +# location: str +# materialized_examples_format: str [Default: 'tfrecords_gzip'] +# max_failed_trial_count: int [Default: 0.0] +# max_selected_features: int [Default: -1.0] +# max_trial_count: int +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# parallel_trial_count: int +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# root_dir: str +# run_evaluation: bool [Default: False] +# run_feature_selection: bool [Default: False] +# seed: int [Default: 1.0] +# stratified_split_key: str [Default: ''] +# study_spec_algorithm: str [Default: 'ALGORITHM_UNSPECIFIED'] +# study_spec_measurement_selection_type: str [Default: 'BEST_MEASUREMENT'] +# study_spec_metric_goal: str +# study_spec_metric_id: str +# study_spec_parameters_override: list +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# tf_custom_transformation_definitions: list +# tf_transform_execution_engine: str [Default: ''] +# tf_transformations_path: str [Default: ''] +# training_fraction: float [Default: -1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# worker_pool_specs_override: list +# Outputs: +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-infra-validator: + executorLabel: exec-automl-tabular-infra-validator + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - model-evaluation + inputs: + artifacts: + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: TabNet Hyperparameter Tuning + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import + inputDefinitions: + artifacts: + pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + automl-tabular-infra-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator + dependentTasks: + - get-best-hyperparameter-tuning-job-trial + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + taskInfo: + name: automl-tabular-infra-validator + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - bool-identity + - feature-transform-engine + - get-best-hyperparameter-tuning-job-trial + - model-upload + inputs: + artifacts: + pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: pipelinechannel--dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: pipelinechannel--dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: pipelinechannel--feature_selection_algorithm + location: + componentInputParameter: pipelinechannel--location + materialized_examples_format: + componentInputParameter: pipelinechannel--materialized_examples_format + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + model_type: + runtimeValue: + constant: neural_network + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_feature_selection: + componentInputParameter: pipelinechannel--run_feature_selection + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--tf_auto_transform_features + tf_custom_transformation_definitions: + componentInputParameter: pipelinechannel--tf_custom_transformation_definitions + tf_transform_execution_engine: + componentInputParameter: pipelinechannel--tf_transform_execution_engine + tf_transformations_path: + componentInputParameter: pipelinechannel--tf_transformations_path + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + get-best-hyperparameter-tuning-job-trial: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-best-hyperparameter-tuning-job-trial + dependentTasks: + - tabnet-hyperparameter-tuning-job + inputs: + parameters: + gcp_resources: + taskOutputParameter: + outputParameterKey: gcp_resources + producerTask: tabnet-hyperparameter-tuning-job + instance_schema_uri: + taskOutputParameter: + outputParameterKey: instance_schema_uri + producerTask: tabnet-hyperparameter-tuning-job + prediction_docker_uri: + taskOutputParameter: + outputParameterKey: prediction_docker_uri_output + producerTask: tabnet-hyperparameter-tuning-job + prediction_schema_uri: + taskOutputParameter: + outputParameterKey: prediction_schema_uri + producerTask: tabnet-hyperparameter-tuning-job + study_spec_metric_goal: + componentInputParameter: pipelinechannel--study_spec_metric_goal + trials_dir: + taskOutputParameter: + outputParameterKey: trials + producerTask: tabnet-hyperparameter-tuning-job + taskInfo: + name: get-best-hyperparameter-tuning-job-trial + get-tabnet-study-spec-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-tabnet-study-spec-parameters + dependentTasks: + - training-configurator-and-validator + inputs: + artifacts: + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + parameters: + max_trial_count: + componentInputParameter: pipelinechannel--max_trial_count + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + taskInfo: + name: get-tabnet-study-spec-parameters + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-tabular-infra-validator + - get-best-hyperparameter-tuning-job-trial + inputs: + artifacts: + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + parse-worker-pool-specs-override: + cachingOptions: + enableCache: true + componentRef: + name: comp-parse-worker-pool-specs-override + inputs: + parameters: + worker_pool_specs_override: + componentInputParameter: pipelinechannel--worker_pool_specs_override + taskInfo: + name: parse-worker-pool-specs-override + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + tabnet-hyperparameter-tuning-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-tabnet-hyperparameter-tuning-job + dependentTasks: + - feature-transform-engine + - get-tabnet-study-spec-parameters + - parse-worker-pool-specs-override + - split-materialized-data + - training-configurator-and-validator + inputs: + artifacts: + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + training_schema_uri: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + parameters: + cache_data: + componentInputParameter: pipelinechannel--cache_data + enable_profiler: + componentInputParameter: pipelinechannel--enable_profiler + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + eval_frequency_secs: + componentInputParameter: pipelinechannel--eval_frequency_secs + eval_steps: + componentInputParameter: pipelinechannel--eval_steps + location: + componentInputParameter: pipelinechannel--location + max_failed_trial_count: + componentInputParameter: pipelinechannel--max_failed_trial_count + max_trial_count: + componentInputParameter: pipelinechannel--max_trial_count + parallel_trial_count: + componentInputParameter: pipelinechannel--parallel_trial_count + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + seed: + componentInputParameter: pipelinechannel--seed + study_spec_algorithm: + componentInputParameter: pipelinechannel--study_spec_algorithm + study_spec_measurement_selection_type: + componentInputParameter: pipelinechannel--study_spec_measurement_selection_type + study_spec_metric_goal: + componentInputParameter: pipelinechannel--study_spec_metric_goal + study_spec_metric_id: + componentInputParameter: pipelinechannel--study_spec_metric_id + study_spec_parameters_override: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-tabnet-study-spec-parameters + target_column: + componentInputParameter: pipelinechannel--target_column + training_disk_spec: + taskOutputParameter: + outputParameterKey: training_disk_spec + producerTask: parse-worker-pool-specs-override + training_machine_spec: + taskOutputParameter: + outputParameterKey: training_machine_spec + producerTask: parse-worker-pool-specs-override + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: tabnet-hyperparameter-tuning-job + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--cache_data: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--dataset_level_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--dataset_level_transformations: + parameterType: LIST + pipelinechannel--enable_profiler: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--eval_frequency_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--eval_steps: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_selection_algorithm: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--materialized_examples_format: + parameterType: STRING + pipelinechannel--max_failed_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--max_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--parallel_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--run_feature_selection: + parameterType: BOOLEAN + pipelinechannel--seed: + parameterType: NUMBER_INTEGER + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--study_spec_algorithm: + parameterType: STRING + pipelinechannel--study_spec_measurement_selection_type: + parameterType: STRING + pipelinechannel--study_spec_metric_goal: + parameterType: STRING + pipelinechannel--study_spec_metric_id: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--tf_auto_transform_features: + parameterType: STRUCT + pipelinechannel--tf_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--tf_transform_execution_engine: + parameterType: STRING + pipelinechannel--tf_transformations_path: + parameterType: STRING + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--worker_pool_specs_override: + parameterType: LIST + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-get-best-hyperparameter-tuning-job-trial: + executorLabel: exec-get-best-hyperparameter-tuning-job-trial + inputDefinitions: + parameters: + gcp_resources: + description: Proto tracking the hyperparameter tuning job. + parameterType: STRING + instance_schema_uri: + defaultValue: '' + description: The instance schema uri. + isOptional: true + parameterType: STRING + prediction_docker_uri: + defaultValue: '' + description: The prediction docker container uri. + isOptional: true + parameterType: STRING + prediction_schema_uri: + defaultValue: '' + description: The prediction schema_uri. + isOptional: true + parameterType: STRING + read_value_from_file: + defaultValue: false + description: If true, read file to get the relevant value. + isOptional: true + parameterType: BOOLEAN + study_spec_metric_goal: + description: 'Optimization goal of the metric, possible values: + + "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + trials_dir: + defaultValue: '' + description: The path to the hyperparameter tuning trials. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-get-tabnet-study-spec-parameters: + executorLabel: exec-get-tabnet-study-spec-parameters + inputDefinitions: + artifacts: + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Metadata generated by example gen. + parameters: + max_trial_count: + description: The desired total number of trials. + parameterType: NUMBER_INTEGER + prediction_type: + description: 'The type of prediction the model is to produce. + + ''classification'' or ''regression''.' + parameterType: STRING + study_spec_parameters_override: + description: 'List of dictionaries representing parameters + + to optimize. The dictionary key is the parameter_id, which is passed to + + training job as a command line argument, and the dictionary value is the + + parameter specification of the metric.' + parameterType: LIST + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: An artifact of a model which to upload a new version to. Only + specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: "The unmanaged container model to be uploaded. The Model can\ + \ be passed from an upstream step or imported via a KFP `dsl.importer`.\n\ + :Examples:\n ::\n\n from kfp import dsl\n from google_cloud_pipeline_components.google_cloud_pipeline_components.types\ + \ import artifact_types\n\n importer_spec = dsl.importer(\n artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model',\n\ + \ artifact_class=artifact_types.UnmanagedContainerModel,\n metadata={\n\ + \ 'containerSpec': { 'imageUri':\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod'\n\ + \ }\n })" + isOptional: true + parameters: + description: + defaultValue: '' + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + isOptional: true + parameterType: STRING + display_name: + description: 'The display name of the Model. The name + + can be up to 128 characters long and can be consist of any UTF-8 + + characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model)' + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key spec for a Model. If set, this Model and all sub-resources of this + + Model will be secured by this key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + description: 'Metadata describing the Model''s + + input and output for explanation. Both `explanation_metadata` and `explanation_parameters` + must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata)' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters)' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your model. Label keys and values can be no longer than 64 + + characters (Unicode codepoints), can only contain lowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. See https://goo.gl/xmQnxf for more information and + + examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Optional location to upload this Model to. If + + not set, defaults to `us-central1`.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to upload this Model to. Defaults to the project in + which the PipelineJob is run. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: Artifact tracking the created Model. + parameters: + gcp_resources: + description: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) + which tracks the upload Model's long-running operation. + parameterType: STRING + comp-parse-worker-pool-specs-override: + executorLabel: exec-parse-worker-pool-specs-override + inputDefinitions: + parameters: + worker_pool_specs_override: + description: 'The list of dictionaries for overriding training + + and evaluation worker pool specs.' + parameterType: LIST + outputDefinitions: + parameters: + eval_machine_spec: + description: The eval machine spec. + parameterType: STRUCT + eval_replica_count: + description: The replica count for eval. + parameterType: NUMBER_INTEGER + training_disk_spec: + description: The training disk spec. + parameterType: STRUCT + training_machine_spec: + description: The training machine spec. + parameterType: STRUCT + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-tabnet-hyperparameter-tuning-job: + executorLabel: exec-tabnet-hyperparameter-tuning-job + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to a JSON file for baseline values. + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized validation split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized train split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Amount of time in seconds to run the trainer for. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to transform output. + parameters: + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to + + ''auto'', caching is determined based on the dataset size.' + isOptional: true + parameterType: STRING + enable_profiler: + defaultValue: false + description: 'Enables profiling and saves a trace + + during evaluation.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and + + checkpointing will take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not + + specified or negative, it means run evaluation on the whole validation + + dataset. If set to 0, it means run evaluation for a fixed number of + + samples.' + isOptional: true + parameterType: NUMBER_INTEGER + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + max_failed_trial_count: + defaultValue: 0.0 + description: 'The number of failed trials that + + need to be seen before failing the HyperparameterTuningJob. If set to + 0, + + Vertex AI decides how many trials must fail before the whole job fails.' + isOptional: true + parameterType: NUMBER_INTEGER + max_trial_count: + description: The desired total number of trials. + parameterType: NUMBER_INTEGER + parallel_trial_count: + description: 'The desired number of trials to run + + in parallel.' + parameterType: NUMBER_INTEGER + prediction_type: + description: 'The type of prediction the model is to + + produce. "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + study_spec_algorithm: + defaultValue: ALGORITHM_UNSPECIFIED + description: 'The search algorithm specified for + + the study. One of ''ALGORITHM_UNSPECIFIED'', ''GRID_SEARCH'', or + + ''RANDOM_SEARCH''.' + isOptional: true + parameterType: STRING + study_spec_measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: 'Which measurement + + to use if/when the service automatically selects the final measurement + + from previously reported intermediate measurements. One of + + "BEST_MEASUREMENT" or "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + study_spec_metric_goal: + description: 'Optimization goal of the metric, + + possible values: "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + study_spec_metric_id: + description: 'Metric to optimize, possible + + values: [ ''loss'', ''average_loss'', ''rmse'', ''mae'', ''mql'', ''accuracy'', + ''auc'', ''precision'', ''recall''].' + parameterType: STRING + study_spec_parameters_override: + description: 'List of dictionaries + + representing parameters to optimize. The dictionary key is the + + parameter_id, which is passed to training job as a command line + + argument, and the dictionary value is the parameter specification of the + + metric.' + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + training_disk_spec: + defaultValue: + boot_disk_size_gb: 100.0 + boot_disk_type: pd-ssd + description: The training disk spec. + isOptional: true + parameterType: STRUCT + training_machine_spec: + defaultValue: + machine_type: c2-standard-16 + description: 'The training machine + + spec. See https://cloud.google.com/compute/docs/machine-types for + + options.' + isOptional: true + parameterType: STRUCT + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + execution_metrics: + description: Core metrics in dictionary of hyperparameter tuning job execution. + parameterType: STRUCT + gcp_resources: + description: Serialized gcp_resources proto tracking the custom training + job. + parameterType: STRING + instance_schema_uri: + description: The path to the instance schema. + parameterType: STRING + prediction_docker_uri_output: + description: The URI of the prediction container. + parameterType: STRING + prediction_schema_uri: + description: The path to the prediction schema. + parameterType: STRING + trials: + description: The path to the hyperparameter tuning trials + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-infra-validator: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-get-best-hyperparameter-tuning-job-trial: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_best_hyperparameter_tuning_job_trial + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_best_hyperparameter_tuning_job_trial(\n gcp_resources:\ + \ str,\n study_spec_metric_goal: str,\n unmanaged_container_model:\ + \ dsl.Output[dsl.Artifact],\n trials_dir: str = '',\n instance_schema_uri:\ + \ str = '',\n prediction_schema_uri: str = '',\n prediction_docker_uri:\ + \ str = '',\n read_value_from_file: bool = False,\n):\n \"\"\"Gets best\ + \ HyperparameterTuningJob trial.\n\n Args:\n gcp_resources: Proto tracking\ + \ the hyperparameter tuning job.\n study_spec_metric_goal: Optimization\ + \ goal of the metric, possible values:\n \"MAXIMIZE\", \"MINIMIZE\"\ + .\n unmanaged_container_model: The unmanaged model.\n trials_dir:\ + \ The path to the hyperparameter tuning trials.\n instance_schema_uri:\ + \ The instance schema uri.\n prediction_schema_uri: The prediction schema_uri.\n\ + \ prediction_docker_uri: The prediction docker container uri.\n read_value_from_file:\ + \ If true, read file to get the relevant value.\n\n Raises:\n RuntimeError:\ + \ If there are multiple metrics.\n \"\"\"\n\n import os\n import json\n\ + \ from google.api_core.retry import Retry\n from google.cloud import aiplatform_v1beta1\ + \ as aip\n import tensorflow as tf\n\n # If path to file with value is\ + \ provided, read the file before continuing.\n if read_value_from_file:\n\ + \ with tf.io.gfile.GFile(trials_dir, 'r') as f:\n trials_dir = f.read()\n\ + \ with tf.io.gfile.GFile(instance_schema_uri, 'r') as f:\n instance_schema_uri\ + \ = f.read()\n with tf.io.gfile.GFile(prediction_schema_uri, 'r') as\ + \ f:\n prediction_schema_uri = f.read()\n with tf.io.gfile.GFile(prediction_docker_uri,\ + \ 'r') as f:\n prediction_docker_uri = f.read()\n\n api_endpoint_suffix\ + \ = '-aiplatform.googleapis.com'\n gcp_resources_json = json.loads(gcp_resources)\n\ + \ resource = gcp_resources_json['resources'][0]\n\n uri_key = 'resource_uri'\n\ + \ if uri_key not in resource:\n uri_key = 'resourceUri'\n\n gcp_resources_split\ + \ = resource[uri_key].partition('projects')\n resource_name = gcp_resources_split[1]\ + \ + gcp_resources_split[2]\n prefix_str = gcp_resources_split[0]\n prefix_str\ + \ = prefix_str[: prefix_str.find(api_endpoint_suffix)]\n api_endpoint =\ + \ (\n prefix_str[(prefix_str.rfind('//') + 2) :] + api_endpoint_suffix\n\ + \ )\n\n job_client = aip.JobServiceClient(\n client_options={'api_endpoint':\ + \ api_endpoint}\n )\n response = job_client.get_hyperparameter_tuning_job(\n\ + \ name=resource_name,\n retry=Retry(initial=10.0, maximum=60.0,\ + \ deadline=10.0 * 60.0),\n )\n\n # Get best trial\n trials_list = []\n\ + \ for trial in response.trials:\n if trial.final_measurement:\n \ + \ trials_list.append({\n 'id': trial.id,\n 'objective_value':\ + \ trial.final_measurement.metrics[0].value,\n })\n\n if study_spec_metric_goal\ + \ == 'MAXIMIZE':\n best_fn = max\n elif study_spec_metric_goal == 'MINIMIZE':\n\ + \ best_fn = min\n else:\n raise ValueError(\n f'Unexpected\ + \ study spec metric goal: {study_spec_metric_goal}'\n )\n\n best_trial\ + \ = best_fn(trials_list, key=lambda trial: trial['objective_value'])\n\n\ + \ # Build unmanaged_container_model\n unmanaged_container_model.metadata['containerSpec']\ + \ = {\n 'imageUri': prediction_docker_uri,\n 'healthRoute': '/health',\n\ + \ 'predictRoute': '/predict',\n }\n unmanaged_container_model.metadata['predictSchemata']\ + \ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\ + \ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\ + \ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-tabnet-study-spec-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_tabnet_study_spec_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_tabnet_study_spec_parameters(\n metadata: dsl.InputPath('TabularExampleGenMetadata'),\n\ + \ max_trial_count: int,\n prediction_type: str,\n study_spec_parameters_override:\ + \ list, # Required for KFP validation; pylint:disable=g-bare-generic\n\ + ) -> list: # Required by KFP; pylint:disable=g-bare-generic\n \"\"\"Get\ + \ study_spec_parameters for a TabNet hyperparameter tuning job.\n\n Args:\n\ + \ metadata: Metadata generated by example gen.\n max_trial_count:\ + \ The desired total number of trials.\n prediction_type: The type of\ + \ prediction the model is to produce.\n 'classification' or 'regression'.\n\ + \ study_spec_parameters_override: List of dictionaries representing parameters\n\ + \ to optimize. The dictionary key is the parameter_id, which is passed\ + \ to\n training job as a command line argument, and the dictionary\ + \ value is the\n parameter specification of the metric.\n\n Returns:\n\ + \ List of final Vizier study_spec_parameters of type ParameterSpec.\n\ + \ \"\"\"\n # Define different search space constants\n tabnet_params_small_data_small_search_space\ + \ = [\n {\n 'parameter_id': 'max_steps',\n 'discrete_value_spec':\ + \ {\n 'values': [5000, 10000, 15000, 20000, 25000, 30000]\n\ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [512, 1024, 2048, 4096]},\n },\n {\n 'parameter_id':\ + \ 'learning_rate',\n 'double_value_spec': {'min_value': 0.0001,\ + \ 'max_value': 0.02},\n 'scale_type': 'UNIT_LOG_SCALE',\n \ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [5]},\n },\n {\n \ + \ 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 50, 'max_value': 200},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.3, 'max_value': 0.7},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 6},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.2,\ + \ 'max_value': 2.5},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 1000, 'max_value': 5000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'sparsity_loss_weight',\n 'double_value_spec': {'min_value':\ + \ 0.000001, 'max_value': 0.001},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'batch_momentum',\n \ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'batch_size_ratio',\n 'discrete_value_spec': {'values': [0.125,\ + \ 0.25, 0.5]},\n },\n {\n 'parameter_id': 'num_transformer_layers',\n\ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 4},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.3, 'max_value': 0.7},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'class_weight',\n \ + \ 'double_value_spec': {'min_value': 1.0, 'max_value': 100.0},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'loss_function_type',\n 'categorical_value_spec': {'values':\ + \ ['weighted_cross_entropy']},\n },\n {\n 'parameter_id':\ + \ 'yeo_johnson_transform',\n 'categorical_value_spec': {'values':\ + \ ['false']},\n },\n ]\n tabnet_params_small_data_medium_search_space\ + \ = [\n {\n 'parameter_id': 'max_steps',\n 'discrete_value_spec':\ + \ {\n 'values': [5000, 10000, 20000, 30000, 40000, 50000]\n\ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [512, 1024, 2048, 4096]},\n },\n {\n 'parameter_id':\ + \ 'learning_rate',\n 'double_value_spec': {'min_value': 0.0001,\ + \ 'max_value': 0.03},\n 'scale_type': 'UNIT_LOG_SCALE',\n \ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [5]},\n },\n {\n \ + \ 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 50, 'max_value': 200},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 6},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.2,\ + \ 'max_value': 2.7},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 1000, 'max_value': 10000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'sparsity_loss_weight',\n 'double_value_spec': {'min_value':\ + \ 0.000001, 'max_value': 0.001},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'batch_momentum',\n \ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'batch_size_ratio',\n 'discrete_value_spec': {'values': [0.0625,\ + \ 0.125, 0.25, 0.5]},\n },\n {\n 'parameter_id': 'num_transformer_layers',\n\ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 6},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'class_weight',\n \ + \ 'double_value_spec': {'min_value': 1.0, 'max_value': 100.0},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'loss_function_type',\n 'categorical_value_spec': {\n \ + \ 'values': ['weighted_cross_entropy', 'focal_loss']\n },\n\ + \ },\n {\n 'parameter_id': 'alpha_focal_loss',\n \ + \ 'discrete_value_spec': {'values': [0.1, 0.25, 0.5, 0.75, 0.9, 0.99]},\n\ + \ },\n {\n 'parameter_id': 'gamma_focal_loss',\n \ + \ 'discrete_value_spec': {'values': [0.0, 0.5, 1.0, 2.0, 3.0, 4.0]},\n\ + \ },\n {\n 'parameter_id': 'yeo_johnson_transform',\n\ + \ 'categorical_value_spec': {'values': ['false']},\n },\n\ + \ ]\n tabnet_params_small_data_large_search_space = [\n {\n \ + \ 'parameter_id': 'max_steps',\n 'discrete_value_spec': {\n\ + \ 'values': [10000, 20000, 30000, 40000, 50000]\n \ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n \ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [512, 1024, 2048, 4096]},\n },\n {\n 'parameter_id':\ + \ 'learning_rate',\n 'double_value_spec': {'min_value': 0.00007,\ + \ 'max_value': 0.03},\n 'scale_type': 'UNIT_LOG_SCALE',\n \ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [3, 5, 10]},\n },\n {\n\ + \ 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [5, 10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 50, 'max_value': 300},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 7},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.05,\ + \ 'max_value': 3.2},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 1000, 'max_value': 10000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'sparsity_loss_weight',\n 'double_value_spec': {'min_value':\ + \ 0.0000001, 'max_value': 0.001},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'batch_momentum',\n \ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'batch_size_ratio',\n 'discrete_value_spec': {'values': [0.0625,\ + \ 0.125, 0.25, 0.5]},\n },\n {\n 'parameter_id': 'num_transformer_layers',\n\ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 6},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'class_weight',\n \ + \ 'double_value_spec': {'min_value': 1.0, 'max_value': 100.0},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'loss_function_type',\n 'categorical_value_spec': {\n \ + \ 'values': ['weighted_cross_entropy', 'focal_loss']\n },\n\ + \ },\n {\n 'parameter_id': 'alpha_focal_loss',\n \ + \ 'discrete_value_spec': {'values': [0.1, 0.25, 0.5, 0.75, 0.9, 0.99]},\n\ + \ },\n {\n 'parameter_id': 'gamma_focal_loss',\n \ + \ 'discrete_value_spec': {'values': [0.0, 0.5, 1.0, 2.0, 3.0, 4.0]},\n\ + \ },\n {\n 'parameter_id': 'yeo_johnson_transform',\n\ + \ 'categorical_value_spec': {'values': ['false', 'true']},\n \ + \ },\n ]\n tabnet_params_medium_data_small_search_space = [\n \ + \ {\n 'parameter_id': 'max_steps',\n 'discrete_value_spec':\ + \ {\n 'values': [10000, 20000, 30000, 40000, 50000]\n \ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [1024, 4096, 8192, 16384]},\n },\n {\n 'parameter_id':\ + \ 'learning_rate',\n 'double_value_spec': {'min_value': 0.0001,\ + \ 'max_value': 0.02},\n 'scale_type': 'UNIT_LOG_SCALE',\n \ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [5]},\n },\n {\n \ + \ 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 100, 'max_value': 300},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.3, 'max_value': 0.7},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 6},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.2,\ + \ 'max_value': 2.5},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 10000, 'max_value': 50000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'sparsity_loss_weight',\n 'double_value_spec':\ + \ {'min_value': 0.0000001, 'max_value': 0.001},\n 'scale_type':\ + \ 'UNIT_LOG_SCALE',\n },\n {\n 'parameter_id': 'batch_momentum',\n\ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'batch_size_ratio',\n 'discrete_value_spec':\ + \ {'values': [0.0625, 0.125, 0.25, 0.5]},\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers',\n 'integer_value_spec':\ + \ {'min_value': 2, 'max_value': 6},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_transformer_layers_ratio',\n\ + \ 'double_value_spec': {'min_value': 0.3, 'max_value': 0.7},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'class_weight',\n 'double_value_spec': {'min_value':\ + \ 1.0, 'max_value': 100.0},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'loss_function_type',\n \ + \ 'categorical_value_spec': {'values': ['weighted_cross_entropy']},\n\ + \ },\n {\n 'parameter_id': 'yeo_johnson_transform',\n\ + \ 'categorical_value_spec': {'values': ['false']},\n },\n\ + \ ]\n tabnet_params_medium_data_medium_search_space = [\n {\n \ + \ 'parameter_id': 'max_steps',\n 'discrete_value_spec': {\n\ + \ 'values': [5000, 10000, 20000, 30000, 40000, 50000]\n \ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [1024, 2048, 4096, 8192, 16384]},\n },\n {\n \ + \ 'parameter_id': 'learning_rate',\n 'double_value_spec':\ + \ {'min_value': 0.00007, 'max_value': 0.02},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [5]},\n },\n {\n \ + \ 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 50, 'max_value': 400},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 6},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.2,\ + \ 'max_value': 2.5},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 10000, 'max_value': 50000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'sparsity_loss_weight',\n 'double_value_spec':\ + \ {'min_value': 0.0000001, 'max_value': 0.001},\n 'scale_type':\ + \ 'UNIT_LOG_SCALE',\n },\n {\n 'parameter_id': 'batch_momentum',\n\ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'batch_size_ratio',\n 'discrete_value_spec':\ + \ {'values': [0.0625, 0.125, 0.25, 0.5]},\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers',\n 'integer_value_spec':\ + \ {'min_value': 4, 'max_value': 10},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_transformer_layers_ratio',\n\ + \ 'double_value_spec': {'min_value': 0.2, 'max_value': 0.8},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'class_weight',\n 'double_value_spec': {'min_value':\ + \ 1.0, 'max_value': 100.0},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'loss_function_type',\n \ + \ 'categorical_value_spec': {\n 'values': ['weighted_cross_entropy',\ + \ 'focal_loss']\n },\n },\n {\n 'parameter_id':\ + \ 'alpha_focal_loss',\n 'discrete_value_spec': {'values': [0.1,\ + \ 0.25, 0.5, 0.75, 0.9, 0.99]},\n },\n {\n 'parameter_id':\ + \ 'gamma_focal_loss',\n 'discrete_value_spec': {'values': [0.0,\ + \ 0.5, 1.0, 2.0, 3.0, 4.0]},\n },\n {\n 'parameter_id':\ + \ 'yeo_johnson_transform',\n 'categorical_value_spec': {'values':\ + \ ['false']},\n },\n ]\n tabnet_params_medium_data_large_search_space\ + \ = [\n {\n 'parameter_id': 'max_steps',\n 'discrete_value_spec':\ + \ {\n 'values': [50000, 60000, 70000, 80000, 90000, 100000]\n\ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [1024, 2048, 4096, 8192, 16384]},\n },\n {\n \ + \ 'parameter_id': 'learning_rate',\n 'double_value_spec':\ + \ {'min_value': 0.00007, 'max_value': 0.03},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [3, 5, 10]},\n },\n \ + \ {\n 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [5, 10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 50, 'max_value': 500},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 2, 'max_value': 8},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.05,\ + \ 'max_value': 3.2},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 10000, 'max_value': 50000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'sparsity_loss_weight',\n 'double_value_spec':\ + \ {'min_value': 0.0000001, 'max_value': 0.001},\n 'scale_type':\ + \ 'UNIT_LOG_SCALE',\n },\n {\n 'parameter_id': 'batch_momentum',\n\ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'batch_size_ratio',\n 'discrete_value_spec':\ + \ {'values': [0.0625, 0.125, 0.25, 0.5]},\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers',\n 'integer_value_spec':\ + \ {'min_value': 4, 'max_value': 10},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_transformer_layers_ratio',\n\ + \ 'double_value_spec': {'min_value': 0.2, 'max_value': 0.8},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'class_weight',\n 'double_value_spec': {'min_value':\ + \ 1.0, 'max_value': 100.0},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'loss_function_type',\n \ + \ 'categorical_value_spec': {\n 'values': ['weighted_cross_entropy',\ + \ 'focal_loss']\n },\n },\n {\n 'parameter_id':\ + \ 'alpha_focal_loss',\n 'discrete_value_spec': {'values': [0.1,\ + \ 0.25, 0.5, 0.75, 0.9, 0.99]},\n },\n {\n 'parameter_id':\ + \ 'gamma_focal_loss',\n 'discrete_value_spec': {'values': [0.0,\ + \ 0.5, 1.0, 2.0, 3.0, 4.0]},\n },\n {\n 'parameter_id':\ + \ 'yeo_johnson_transform',\n 'categorical_value_spec': {'values':\ + \ ['false', 'true']},\n },\n ]\n tabnet_params_large_data_small_search_space\ + \ = [\n {\n 'parameter_id': 'max_steps',\n 'discrete_value_spec':\ + \ {\n 'values': [50000, 60000, 70000, 80000, 90000, 100000]\n\ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [8192, 16384, 32768]},\n },\n {\n 'parameter_id':\ + \ 'learning_rate',\n 'double_value_spec': {'min_value': 0.0002,\ + \ 'max_value': 0.02},\n 'scale_type': 'UNIT_LOG_SCALE',\n \ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [5]},\n },\n {\n \ + \ 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 100, 'max_value': 400},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 3, 'max_value': 6},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.2,\ + \ 'max_value': 2.5},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 10000, 'max_value': 50000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'sparsity_loss_weight',\n 'double_value_spec':\ + \ {'min_value': 0.000001, 'max_value': 0.001},\n 'scale_type':\ + \ 'UNIT_LOG_SCALE',\n },\n {\n 'parameter_id': 'batch_momentum',\n\ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'batch_size_ratio',\n 'discrete_value_spec':\ + \ {'values': [0.0625, 0.125, 0.25, 0.5]},\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers',\n 'integer_value_spec':\ + \ {'min_value': 2, 'max_value': 6},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_transformer_layers_ratio',\n\ + \ 'double_value_spec': {'min_value': 0.3, 'max_value': 0.7},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'class_weight',\n 'double_value_spec': {'min_value':\ + \ 1.0, 'max_value': 10.0},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'loss_function_type',\n \ + \ 'categorical_value_spec': {'values': ['weighted_cross_entropy']},\n\ + \ },\n {\n 'parameter_id': 'yeo_johnson_transform',\n\ + \ 'categorical_value_spec': {'values': ['false']},\n },\n\ + \ ]\n tabnet_params_large_data_medium_search_space = [\n {\n \ + \ 'parameter_id': 'max_steps',\n 'discrete_value_spec': {\n\ + \ 'values': [50000, 60000, 70000, 80000, 90000, 100000]\n \ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [4096, 8192, 16384, 32768]},\n },\n {\n \ + \ 'parameter_id': 'learning_rate',\n 'double_value_spec': {'min_value':\ + \ 0.0001, 'max_value': 0.03},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [5]},\n },\n {\n \ + \ 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 200, 'max_value': 500},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 3, 'max_value': 7},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.2,\ + \ 'max_value': 2.5},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 10000, 'max_value': 50000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'sparsity_loss_weight',\n 'double_value_spec':\ + \ {'min_value': 0.000001, 'max_value': 0.001},\n 'scale_type':\ + \ 'UNIT_LOG_SCALE',\n },\n {\n 'parameter_id': 'batch_momentum',\n\ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'batch_size_ratio',\n 'discrete_value_spec':\ + \ {'values': [0.0625, 0.125, 0.25, 0.5]},\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers',\n 'integer_value_spec':\ + \ {'min_value': 4, 'max_value': 8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_transformer_layers_ratio',\n\ + \ 'double_value_spec': {'min_value': 0.2, 'max_value': 0.8},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'class_weight',\n 'double_value_spec': {'min_value':\ + \ 1.0, 'max_value': 100.0},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'loss_function_type',\n \ + \ 'categorical_value_spec': {\n 'values': ['weighted_cross_entropy',\ + \ 'focal_loss']\n },\n },\n {\n 'parameter_id':\ + \ 'alpha_focal_loss',\n 'discrete_value_spec': {'values': [0.1,\ + \ 0.25, 0.5, 0.75, 0.9, 0.99]},\n },\n {\n 'parameter_id':\ + \ 'gamma_focal_loss',\n 'discrete_value_spec': {'values': [0.0,\ + \ 0.5, 1.0, 2.0, 3.0, 4.0]},\n },\n {\n 'parameter_id':\ + \ 'yeo_johnson_transform',\n 'categorical_value_spec': {'values':\ + \ ['false']},\n },\n ]\n tabnet_params_large_data_large_search_space\ + \ = [\n {\n 'parameter_id': 'max_steps',\n 'discrete_value_spec':\ + \ {\n 'values': [50000, 70000, 90000, 110000, 130000, 150000]\n\ + \ },\n },\n {\n 'parameter_id': 'max_train_secs',\n\ + \ 'discrete_value_spec': {'values': [-1]},\n },\n {\n\ + \ 'parameter_id': 'batch_size',\n 'discrete_value_spec':\ + \ {'values': [4096, 8192, 16384, 32768, 65536]},\n },\n {\n \ + \ 'parameter_id': 'learning_rate',\n 'double_value_spec':\ + \ {'min_value': 0.00007, 'max_value': 0.03},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'large_category_dim',\n \ + \ 'discrete_value_spec': {'values': [3, 5, 10]},\n },\n \ + \ {\n 'parameter_id': 'large_category_thresh',\n 'discrete_value_spec':\ + \ {'values': [5, 10]},\n },\n {\n 'parameter_id': 'feature_dim',\n\ + \ 'integer_value_spec': {'min_value': 100, 'max_value': 700},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'feature_dim_ratio',\n 'double_value_spec':\ + \ {'min_value': 0.2, 'max_value': 0.8},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_decision_steps',\n \ + \ 'integer_value_spec': {'min_value': 3, 'max_value': 8},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n 'parameter_id':\ + \ 'relaxation_factor',\n 'double_value_spec': {'min_value': 1.05,\ + \ 'max_value': 3.2},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n {\n 'parameter_id': 'decay_rate',\n 'double_value_spec':\ + \ {'min_value': 0.5, 'max_value': 0.999},\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n {\n 'parameter_id': 'decay_every',\n \ + \ 'integer_value_spec': {'min_value': 10000, 'max_value': 50000},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'sparsity_loss_weight',\n 'double_value_spec':\ + \ {'min_value': 0.0000001, 'max_value': 0.001},\n 'scale_type':\ + \ 'UNIT_LOG_SCALE',\n },\n {\n 'parameter_id': 'batch_momentum',\n\ + \ 'double_value_spec': {'min_value': 0.5, 'max_value': 0.95},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'batch_size_ratio',\n 'discrete_value_spec':\ + \ {'values': [0.0625, 0.125, 0.25, 0.5]},\n },\n {\n \ + \ 'parameter_id': 'num_transformer_layers',\n 'integer_value_spec':\ + \ {'min_value': 4, 'max_value': 10},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'num_transformer_layers_ratio',\n\ + \ 'double_value_spec': {'min_value': 0.2, 'max_value': 0.8},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'class_weight',\n 'double_value_spec': {'min_value':\ + \ 1.0, 'max_value': 100.0},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'loss_function_type',\n \ + \ 'categorical_value_spec': {\n 'values': ['weighted_cross_entropy',\ + \ 'focal_loss']\n },\n },\n {\n 'parameter_id':\ + \ 'alpha_focal_loss',\n 'discrete_value_spec': {'values': [0.1,\ + \ 0.25, 0.5, 0.75, 0.9, 0.99]},\n },\n {\n 'parameter_id':\ + \ 'gamma_focal_loss',\n 'discrete_value_spec': {'values': [0.0,\ + \ 0.5, 1.0, 2.0, 3.0, 4.0]},\n },\n {\n 'parameter_id':\ + \ 'yeo_johnson_transform',\n 'categorical_value_spec': {'values':\ + \ ['false', 'true']},\n },\n ]\n search_spaces = {\n 'tabnet_params_small_data_small_search_space':\ + \ (\n tabnet_params_small_data_small_search_space\n ),\n \ + \ 'tabnet_params_small_data_medium_search_space': (\n tabnet_params_small_data_medium_search_space\n\ + \ ),\n 'tabnet_params_small_data_large_search_space': (\n \ + \ tabnet_params_small_data_large_search_space\n ),\n 'tabnet_params_medium_data_small_search_space':\ + \ (\n tabnet_params_medium_data_small_search_space\n ),\n\ + \ 'tabnet_params_medium_data_medium_search_space': (\n tabnet_params_medium_data_medium_search_space\n\ + \ ),\n 'tabnet_params_medium_data_large_search_space': (\n \ + \ tabnet_params_medium_data_large_search_space\n ),\n 'tabnet_params_large_data_small_search_space':\ + \ (\n tabnet_params_large_data_small_search_space\n ),\n \ + \ 'tabnet_params_large_data_medium_search_space': (\n tabnet_params_large_data_medium_search_space\n\ + \ ),\n 'tabnet_params_large_data_large_search_space': (\n \ + \ tabnet_params_large_data_large_search_space\n ),\n }\n\n #\ + \ pylint: disable=g-import-not-at-top,import-outside-toplevel\n import\ + \ json\n import warnings\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \ with open(metadata, 'r') as f:\n metadata_path = f.read()\n metadata\ + \ = json.loads(metadata_path)\n # Calculate dataset size bucket. One of\ + \ 'small' (< 1M rows),\n # 'medium' (1M - 100M rows), or 'large' (> 100M\ + \ rows)\n num_rows = (\n metadata['num_examples']['train']\n \ + \ + metadata['num_examples']['valid']\n + metadata['num_examples']['test']\n\ + \ )\n dataset_size_bucket = 'medium'\n if num_rows < 10000000:\n dataset_size_bucket\ + \ = 'small'\n elif num_rows > 100000000:\n dataset_size_bucket = 'large'\n\ + \n # Calculate search space bucket using max_trial_count.\n # One of 'small'\ + \ (< 10), medium (1 - 100), and large (> 100).\n search_space = 'medium'\n\ + \ if max_trial_count < 10:\n search_space = 'small'\n elif max_trial_count\ + \ > 100:\n search_space = 'large'\n\n # Get params for classification.\n\ + \ params = search_spaces[\n f'tabnet_params_{dataset_size_bucket}_data_{search_space}_search_space'\n\ + \ ]\n\n # Format for regression. To get regression study_spec_parameters,\ + \ we need\n # to set `loss_function_type` to \u2018mae\u2019 (\u2018mae\u2019\ + \ and \u2018mse\u2019 for 'large'\n # search space), remove the `alpha_focal_loss`,\ + \ `gamma_focal_loss`\n # and `class_weight` parameters and increase the\ + \ max for\n # `sparsity_loss_weight` to 100.\n if prediction_type == 'regression':\n\ + \ formatted_params = []\n for param in params:\n if param['parameter_id']\ + \ in [\n 'alpha_focal_loss',\n 'gamma_focal_loss',\n \ + \ 'class_weight',\n ]:\n continue\n elif param['parameter_id']\ + \ == 'sparsity_loss_weight':\n param['double_value_spec']['max_value']\ + \ = 100\n elif param['parameter_id'] == 'loss_function_type':\n \ + \ if search_space == 'large':\n param['categorical_value_spec']['values']\ + \ = ['mae', 'mse']\n else:\n param['categorical_value_spec']['values']\ + \ = ['mae']\n formatted_params.append(param)\n else:\n formatted_params\ + \ = params\n\n # Create parameter_id -> parameter_config dictionary for\ + \ params to override\n # and override parameters.\n override_params =\ + \ {}\n for param in study_spec_parameters_override:\n override_params[param['parameter_id']]\ + \ = param\n\n study_spec_parameters = []\n for param in formatted_params:\n\ + \ study_spec_parameters.append(\n override_params.get(param['parameter_id'],\ + \ param)\n )\n\n extra_overrides = set(override_params) - set(\n \ + \ p['parameter_id'] for p in params\n )\n if extra_overrides:\n extra_override_str\ + \ = ', '.join(extra_overrides)\n warnings.warn(\n f'The overrides\ + \ {extra_override_str} were not found in the params and '\n 'will\ + \ be ignored.'\n )\n\n return study_spec_parameters\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"pipeline_job\": + \"", "projects/{{$.inputs.parameters[''project'']}}/locations/{{$.inputs.parameters[''location'']}}/pipelineJobs/{{$.pipeline_job_uuid}}", + "\"", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.model.upload_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-parse-worker-pool-specs-override: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _parse_worker_pool_specs_override + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _parse_worker_pool_specs_override(\n worker_pool_specs_override:\ + \ list, # pylint:disable=g-bare-generic\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('training_machine_spec', dict), # pylint:disable=g-bare-generic\n\ + \ ('training_disk_spec', dict),\n ('eval_machine_spec', dict),\ + \ # pylint:disable=g-bare-generic\n ('eval_replica_count', int),\n\ + \ ],\n):\n \"\"\"Parses worker_pool_specs_override and returns training\ + \ and evaluation machine specifications.\n\n Args:\n worker_pool_specs_override:\ + \ The list of dictionaries for overriding training\n and evaluation\ + \ worker pool specs.\n\n Returns:\n training_machine_spec: The training\ + \ machine spec.\n training_disk_spec: The training disk spec.\n \ + \ eval_machine_spec: The eval machine spec.\n eval_replica_count:\ + \ The replica count for eval.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n training_machine_spec = {'machine_type': 'c2-standard-16'}\n training_disk_spec\ + \ = {'boot_disk_type': 'pd-ssd', 'boot_disk_size_gb': 100}\n eval_machine_spec\ + \ = {'machine_type': 'c2-standard-8'}\n eval_replica_count = 1\n\n if\ + \ worker_pool_specs_override:\n if len(worker_pool_specs_override) >=\ + \ 1 and isinstance(\n worker_pool_specs_override[0], dict\n ):\n\ + \ training_machine_spec = worker_pool_specs_override[0].get(\n \ + \ 'machine_spec', training_machine_spec\n )\n training_disk_spec\ + \ = worker_pool_specs_override[0].get(\n 'disk_spec', training_disk_spec\n\ + \ )\n if len(worker_pool_specs_override) == 4 and isinstance(\n\ + \ worker_pool_specs_override[3], dict\n ):\n eval_machine_spec\ + \ = worker_pool_specs_override[3].get(\n 'machine_spec', eval_machine_spec\n\ + \ )\n eval_replica_count = worker_pool_specs_override[3].get(\n\ + \ 'replica_count', eval_replica_count\n )\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'training_machine_spec',\n \ + \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ + \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ + \ eval_machine_spec,\n eval_replica_count,\n )\n\n" + image: python:3.7 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-tabnet-hyperparameter-tuning-job: + container: + args: + - --type + - HyperparameterTuningJobWithMetrics + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --execution_metrics + - '{{$.outputs.parameters[''execution_metrics''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"tabnet-hyperparameter-tuning-job-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"study_spec\": {\"metrics\": [{\"metric_id\": \"", "{{$.inputs.parameters[''study_spec_metric_id'']}}", + "\", \"goal\": \"", "{{$.inputs.parameters[''study_spec_metric_goal'']}}", + "\"}], \"parameters\": ", "{{$.inputs.parameters[''study_spec_parameters_override'']}}", + ", \"algorithm\": \"", "{{$.inputs.parameters[''study_spec_algorithm'']}}", + "\", \"measurement_selection_type\": \"", "{{$.inputs.parameters[''study_spec_measurement_selection_type'']}}", + "\"}, \"max_trial_count\": ", "{{$.inputs.parameters[''max_trial_count'']}}", + ", \"parallel_trial_count\": ", "{{$.inputs.parameters[''parallel_trial_count'']}}", + ", \"max_failed_trial_count\": ", "{{$.inputs.parameters[''max_failed_trial_count'']}}", + ", \"trial_job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", + "1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", + ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20231002_0125", + "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", + "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", + "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--prediction_docker_uri_artifact_path=", "{{$.outputs.parameters[''prediction_docker_uri_output''].output_file}}", + "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_schema_path=", "{{$.inputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--instance_schema_path=", "{{$.outputs.parameters[''instance_schema_uri''].output_file}}", + "\", \"--prediction_schema_path=", "{{$.outputs.parameters[''prediction_schema_uri''].output_file}}", + "\", \"--trials_path=", "{{$.outputs.parameters[''trials''].output_file}}", + "\", \"--job_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--training_data_path=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--validation_data_path=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--enable_profiler=", "{{$.inputs.parameters[''enable_profiler'']}}", + "\", \"--cache_data=", "{{$.inputs.parameters[''cache_data'']}}", "\", \"--seed=", + "{{$.inputs.parameters[''seed'']}}", "\", \"--measurement_selection_type=", + "{{$.inputs.parameters[''study_spec_measurement_selection_type'']}}", "\", + \"--metric_goal=", "{{$.inputs.parameters[''study_spec_metric_goal'']}}", + "\", \"--eval_steps=", "{{$.inputs.parameters[''eval_steps'']}}", "\", \"--eval_frequency_secs=", + "{{$.inputs.parameters[''eval_frequency_secs'']}}", "\", \"--generate_feature_importance=true\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.hyperparameter_tuning_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: The TabNet built-in algorithm HyperparameterTuningJob pipeline. + name: automl-tabular-tabnet-hyperparameter-tuning-job +root: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + pipelinechannel--cache_data: + componentInputParameter: cache_data + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + pipelinechannel--dataset_level_transformations: + componentInputParameter: dataset_level_transformations + pipelinechannel--enable_profiler: + componentInputParameter: enable_profiler + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--eval_frequency_secs: + componentInputParameter: eval_frequency_secs + pipelinechannel--eval_steps: + componentInputParameter: eval_steps + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--materialized_examples_format: + componentInputParameter: materialized_examples_format + pipelinechannel--max_failed_trial_count: + componentInputParameter: max_failed_trial_count + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--max_trial_count: + componentInputParameter: max_trial_count + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--parallel_trial_count: + componentInputParameter: parallel_trial_count + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--prediction_type: + componentInputParameter: prediction_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--run_feature_selection: + componentInputParameter: run_feature_selection + pipelinechannel--seed: + componentInputParameter: seed + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--study_spec_algorithm: + componentInputParameter: study_spec_algorithm + pipelinechannel--study_spec_measurement_selection_type: + componentInputParameter: study_spec_measurement_selection_type + pipelinechannel--study_spec_metric_goal: + componentInputParameter: study_spec_metric_goal + pipelinechannel--study_spec_metric_id: + componentInputParameter: study_spec_metric_id + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + pipelinechannel--tf_custom_transformation_definitions: + componentInputParameter: tf_custom_transformation_definitions + pipelinechannel--tf_transform_execution_engine: + componentInputParameter: tf_transform_execution_engine + pipelinechannel--tf_transformations_path: + componentInputParameter: tf_transformations_path + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--worker_pool_specs_override: + componentInputParameter: worker_pool_specs_override + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Parent model if this model is uploaded as a version. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + bigquery_staging_full_dataset_id: + defaultValue: '' + description: Staging directory for BigQuery tables. + isOptional: true + parameterType: STRING + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to ''auto'', caching is + + determined based on the dataset size.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + description: 'Dataset-level custom + + transformation definitions in string format.' + isOptional: true + parameterType: LIST + dataset_level_transformations: + description: 'Dataset-level transformation configuration in + + string format.' + isOptional: true + parameterType: LIST + enable_profiler: + defaultValue: false + description: Enables profiling and saves a trace during evaluation. + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and checkpointing will + + take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not specified or + + negative, it means run evaluation on the whole validation dataset. If set + + to 0, it means run evaluation for a fixed number of samples.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_selection_algorithm: + defaultValue: AMI + description: Feature selection algorithm. + isOptional: true + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: The format for the materialized examples. + isOptional: true + parameterType: STRING + max_failed_trial_count: + defaultValue: 0.0 + description: 'The number of failed trials that need to be seen + + before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides + + how many trials must fail before the whole job fails.' + isOptional: true + parameterType: NUMBER_INTEGER + max_selected_features: + defaultValue: -1.0 + description: Maximum number of features to select. + isOptional: true + parameterType: NUMBER_INTEGER + max_trial_count: + description: The desired total number of trials. + parameterType: NUMBER_INTEGER + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + parallel_trial_count: + description: The desired number of trials to run in parallel. + parameterType: NUMBER_INTEGER + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to produce. + + "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: Whether to enable feature selection. + isOptional: true + parameterType: BOOLEAN + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + study_spec_algorithm: + defaultValue: ALGORITHM_UNSPECIFIED + description: 'The search algorithm specified for the study. One of + + ''ALGORITHM_UNSPECIFIED'', ''GRID_SEARCH'', or ''RANDOM_SEARCH''.' + isOptional: true + parameterType: STRING + study_spec_measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: ' Which measurement to use if/when the + + service automatically selects the final measurement from previously + + reported intermediate measurements. One of "BEST_MEASUREMENT" or + + "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + study_spec_metric_goal: + description: 'Optimization goal of the metric, possible values: + + "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + study_spec_metric_id: + description: 'Metric to optimize, possible values: [ ''loss'', + + ''average_loss'', ''rmse'', ''mae'', ''mql'', ''accuracy'', ''auc'', ''precision'', + + ''recall''].' + parameterType: STRING + study_spec_parameters_override: + description: 'List of dictionaries representing parameters + + to optimize. The dictionary key is the parameter_id, which is passed to + + training job as a command line argument, and the dictionary value is the + + parameter specification of the metric.' + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: Test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + description: List of auto transform features. + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + description: 'TF custom transformation definitions + + in string format.' + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: '' + description: 'Execution engine to run TF-based + + transformations. Currently supports "dataflow" or "bigquery"' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: Path to TF transformation configuration. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + validation_fraction: + defaultValue: -1.0 + description: Validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + worker_pool_specs_override: + description: 'The dictionary for overriding training and + + evaluation worker pool specs. The dictionary should be of format + + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py new file mode 100644 index 0000000000..8ffa57a135 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer.py @@ -0,0 +1,261 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Tabnet Trainer component spec.""" + +from typing import Optional + +from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.container_component +def tabnet_trainer( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + learning_rate: float, + instance_baseline: Input[Artifact], + metadata: Input[Artifact], + materialized_train_split: Input[Artifact], + materialized_eval_split: Input[Artifact], + transform_output: Input[Artifact], + training_schema_uri: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + unmanaged_container_model: Output[UnmanagedContainerModel], # pylint: disable=unused-argument + weight_column: Optional[str] = '', + max_steps: Optional[int] = -1, + max_train_secs: Optional[int] = -1, + large_category_dim: Optional[int] = 1, + large_category_thresh: Optional[int] = 300, + yeo_johnson_transform: Optional[bool] = True, + feature_dim: Optional[int] = 64, + feature_dim_ratio: Optional[float] = 0.5, + num_decision_steps: Optional[int] = 6, + relaxation_factor: Optional[float] = 1.5, + decay_every: Optional[float] = 100, + decay_rate: Optional[float] = 0.95, + gradient_thresh: Optional[float] = 2000, + sparsity_loss_weight: Optional[float] = 1e-05, + batch_momentum: Optional[float] = 0.95, + batch_size_ratio: Optional[float] = 0.25, + num_transformer_layers: Optional[int] = 4, + num_transformer_layers_ratio: Optional[float] = 0.25, + class_weight: Optional[float] = 1.0, + loss_function_type: Optional[str] = 'default', + alpha_focal_loss: Optional[float] = 0.25, + gamma_focal_loss: Optional[float] = 2.0, + enable_profiler: Optional[bool] = False, + cache_data: Optional[str] = 'auto', + seed: Optional[int] = 1, + eval_steps: Optional[int] = 0, + batch_size: Optional[int] = 100, + measurement_selection_type: Optional[str] = 'BEST_MEASUREMENT', + optimization_metric: Optional[str] = '', + eval_frequency_secs: Optional[int] = 600, + training_machine_spec: Optional[dict] = {'machine_type': 'c2-standard-16'}, + training_disk_spec: Optional[dict] = { + 'boot_disk_type': 'pd-ssd', + 'boot_disk_size_gb': 100, + }, + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Trains a TabNet model using Vertex CustomJob API. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + weight_column: The weight column name. + max_steps: Number of steps to run the trainer for. + max_train_secs: Amount of time in seconds to run the trainer for. + learning_rate: The learning rate used by the linear optimizer. + large_category_dim: Embedding dimension for categorical feature with large number of categories. + large_category_thresh: Threshold for number of categories to apply large_category_dim embedding dimension to. + yeo_johnson_transform: Enables trainable Yeo-Johnson power transform. + feature_dim: Dimensionality of the hidden representation in feature transformation block. + feature_dim_ratio: The ratio of output dimension (dimensionality of the outputs of each decision step) to feature dimension. + num_decision_steps: Number of sequential decision steps. + relaxation_factor: Relaxation factor that promotes the reuse of each feature at different decision steps. When it is 1, a feature is enforced to be used only at one decision step and as it increases, more flexibility is provided to use a feature at multiple decision steps. + decay_every: Number of iterations for periodically applying learning rate decaying. + decay_rate: Learning rate decaying. + gradient_thresh: Threshold for the norm of gradients for clipping. + sparsity_loss_weight: Weight of the loss for sparsity regularization (increasing it will yield more sparse feature selection). + batch_momentum: Momentum in ghost batch normalization. + batch_size_ratio: The ratio of virtual batch size (size of the ghost batch normalization) to batch size. + num_transformer_layers: The number of transformer layers for each decision step. used only at one decision step and as it increases, more flexibility is provided to use a feature at multiple decision steps. + num_transformer_layers_ratio: The ratio of shared transformer layer to transformer layers. + class_weight: The class weight is used to computes a weighted cross entropy which is helpful in classify imbalanced dataset. Only used for classification. + loss_function_type: Loss function type. Loss function in classification [cross_entropy, weighted_cross_entropy, focal_loss], default is cross_entropy. Loss function in regression: [rmse, mae, mse], default is mse. + alpha_focal_loss: Alpha value (balancing factor) in focal_loss function. Only used for classification. + gamma_focal_loss: Gamma value (modulating factor) for focal loss for focal loss. Only used for classification. + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or negative, it means run evaluation on the whole validation dataset. If set to 0, it means run evaluation for a fixed number of samples. + batch_size: Batch size for training. + measurement_selection_type: Which measurement to use if/when the service automatically selects the final measurement from previously reported intermediate measurements. One of "BEST_MEASUREMENT" or "LAST_MEASUREMENT". + optimization_metric: Optimization metric used for `measurement_selection_type`. Default is "rmse" for regression and "auc" for classification. + eval_frequency_secs: Frequency at which evaluation and checkpointing will take place. + training_machine_spec: The training machine spec. See https://cloud.google.com/compute/docs/machine-types for options. + training_disk_spec: The training disk spec. + instance_baseline: The path to a JSON file for baseline values. + metadata: Amount of time in seconds to run the trainer for. + materialized_train_split: The path to the materialized train split. + materialized_eval_split: The path to the materialized validation split. + transform_output: The path to transform output. + training_schema_uri: The path to the training schema. + encryption_spec_key_name: The KMS key name. + + Returns: + gcp_resources: Serialized gcp_resources proto tracking the custom training job. + unmanaged_container_model: The UnmanagedContainerModel artifact. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "tabnet-trainer-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":"', + '1', + '", "machine_spec": ', + training_machine_spec, + ', "disk_spec": ', + training_disk_spec, + ', "container_spec": {"image_uri":"', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20231002_0125', + '", "args": ["--target_column=', + target_column, + '", "--weight_column=', + weight_column, + '", "--model_type=', + prediction_type, + '", "--prediction_docker_uri=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125', + '", "--baseline_path=', + instance_baseline.uri, + '", "--metadata_path=', + metadata.uri, + '", "--transform_output_path=', + transform_output.uri, + '", "--training_schema_path=', + training_schema_uri.uri, + '", "--job_dir=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train",' + ' "--training_data_path=' + ), + materialized_train_split.uri, + '", "--validation_data_path=', + materialized_eval_split.uri, + '", "--max_steps=', + max_steps, + '", "--max_train_secs=', + max_train_secs, + '", "--learning_rate=', + learning_rate, + '", "--large_category_dim=', + large_category_dim, + '", "--large_category_thresh=', + large_category_thresh, + '", "--yeo_johnson_transform=', + yeo_johnson_transform, + '", "--feature_dim=', + feature_dim, + '", "--feature_dim_ratio=', + feature_dim_ratio, + '", "--num_decision_steps=', + num_decision_steps, + '", "--relaxation_factor=', + relaxation_factor, + '", "--decay_every=', + decay_every, + '", "--decay_rate=', + decay_rate, + '", "--gradient_thresh=', + gradient_thresh, + '", "--sparsity_loss_weight=', + sparsity_loss_weight, + '", "--batch_momentum=', + batch_momentum, + '", "--batch_size_ratio=', + batch_size_ratio, + '", "--num_transformer_layers=', + num_transformer_layers, + '", "--num_transformer_layers_ratio=', + num_transformer_layers_ratio, + '", "--class_weight=', + class_weight, + '", "--loss_function_type=', + loss_function_type, + '", "--alpha_focal_loss=', + alpha_focal_loss, + '", "--gamma_focal_loss=', + gamma_focal_loss, + '", "--enable_profiler=', + enable_profiler, + '", "--cache_data=', + cache_data, + '", "--seed=', + seed, + '", "--eval_steps=', + eval_steps, + '", "--batch_size=', + batch_size, + '", "--measurement_selection_type=', + measurement_selection_type, + '", "--optimization_metric=', + optimization_metric, + '", "--eval_frequency_secs=', + eval_frequency_secs, + ( + '", "--generate_feature_importance=true",' + ' "--executor_input={{$.json_escape[1]}}"]}}]}}' + ), + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml new file mode 100644 index 0000000000..6126b36af9 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/tabnet_trainer_pipeline.yaml @@ -0,0 +1,4413 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-tabnet-trainer +# Description: Train a model using the Tabular Workflow for TabNet pipelines. +# TabNet uses sequential attention to choose which features to reason from at +# each decision step, promoting interpretability and more efficient learning. +# Inputs: +# alpha_focal_loss: float [Default: 0.25] +# batch_momentum: float [Default: 0.95] +# batch_size: int [Default: 100.0] +# batch_size_ratio: float [Default: 0.25] +# bigquery_staging_full_dataset_id: str [Default: ''] +# cache_data: str [Default: 'auto'] +# class_weight: float [Default: 1.0] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# decay_every: float [Default: 100.0] +# decay_rate: float [Default: 0.95] +# enable_profiler: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# eval_frequency_secs: int [Default: 600.0] +# eval_steps: int [Default: 0.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# feature_dim: int [Default: 64.0] +# feature_dim_ratio: float [Default: 0.5] +# feature_selection_algorithm: str [Default: 'AMI'] +# gamma_focal_loss: float [Default: 2.0] +# gradient_thresh: float [Default: 2000.0] +# large_category_dim: int [Default: 1.0] +# large_category_thresh: int [Default: 300.0] +# learning_rate: float +# location: str +# loss_function_type: str [Default: 'default'] +# materialized_examples_format: str [Default: 'tfrecords_gzip'] +# max_selected_features: int [Default: -1.0] +# max_steps: int [Default: -1.0] +# max_train_secs: int [Default: -1.0] +# measurement_selection_type: str [Default: 'BEST_MEASUREMENT'] +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# num_decision_steps: int [Default: 6.0] +# num_transformer_layers: int [Default: 4.0] +# num_transformer_layers_ratio: float [Default: 0.25] +# optimization_metric: str [Default: ''] +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# relaxation_factor: float [Default: 1.5] +# root_dir: str +# run_evaluation: bool [Default: False] +# run_feature_selection: bool [Default: False] +# seed: int [Default: 1.0] +# sparsity_loss_weight: float [Default: 1e-05] +# stratified_split_key: str [Default: ''] +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# tf_custom_transformation_definitions: list +# tf_transform_execution_engine: str [Default: ''] +# tf_transformations_path: str [Default: ''] +# training_fraction: float [Default: -1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# worker_pool_specs_override: list +# yeo_johnson_transform: bool [Default: True] +# Outputs: +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-infra-validator: + executorLabel: exec-automl-tabular-infra-validator + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--tabnet-trainer-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - model-evaluation + inputs: + artifacts: + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_path: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + dataset_type: + runtimeValue: + constant: bigquery + display_name: + runtimeValue: + constant: TabNet Trainer + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import + inputDefinitions: + artifacts: + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + pipelinechannel--tabnet-trainer-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + automl-tabular-infra-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator + dependentTasks: + - tabnet-trainer + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: tabnet-trainer + taskInfo: + name: automl-tabular-infra-validator + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - bool-identity + - feature-transform-engine + - model-upload + - tabnet-trainer + inputs: + artifacts: + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + pipelinechannel--tabnet-trainer-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: tabnet-trainer + parameters: + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: pipelinechannel--dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: pipelinechannel--dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: pipelinechannel--feature_selection_algorithm + location: + componentInputParameter: pipelinechannel--location + materialized_examples_format: + componentInputParameter: pipelinechannel--materialized_examples_format + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + model_type: + runtimeValue: + constant: neural_network + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_feature_selection: + componentInputParameter: pipelinechannel--run_feature_selection + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--tf_auto_transform_features + tf_custom_transformation_definitions: + componentInputParameter: pipelinechannel--tf_custom_transformation_definitions + tf_transform_execution_engine: + componentInputParameter: pipelinechannel--tf_transform_execution_engine + tf_transformations_path: + componentInputParameter: pipelinechannel--tf_transformations_path + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-tabular-infra-validator + - tabnet-trainer + inputs: + artifacts: + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: tabnet-trainer + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + parse-worker-pool-specs-override: + cachingOptions: + enableCache: true + componentRef: + name: comp-parse-worker-pool-specs-override + inputs: + parameters: + worker_pool_specs_override: + componentInputParameter: pipelinechannel--worker_pool_specs_override + taskInfo: + name: parse-worker-pool-specs-override + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + tabnet-trainer: + cachingOptions: + enableCache: true + componentRef: + name: comp-tabnet-trainer + dependentTasks: + - feature-transform-engine + - parse-worker-pool-specs-override + - split-materialized-data + - training-configurator-and-validator + inputs: + artifacts: + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + training_schema_uri: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + parameters: + alpha_focal_loss: + componentInputParameter: pipelinechannel--alpha_focal_loss + batch_momentum: + componentInputParameter: pipelinechannel--batch_momentum + batch_size: + componentInputParameter: pipelinechannel--batch_size + batch_size_ratio: + componentInputParameter: pipelinechannel--batch_size_ratio + cache_data: + componentInputParameter: pipelinechannel--cache_data + class_weight: + componentInputParameter: pipelinechannel--class_weight + decay_every: + componentInputParameter: pipelinechannel--decay_every + decay_rate: + componentInputParameter: pipelinechannel--decay_rate + enable_profiler: + componentInputParameter: pipelinechannel--enable_profiler + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + eval_frequency_secs: + componentInputParameter: pipelinechannel--eval_frequency_secs + eval_steps: + componentInputParameter: pipelinechannel--eval_steps + feature_dim: + componentInputParameter: pipelinechannel--feature_dim + feature_dim_ratio: + componentInputParameter: pipelinechannel--feature_dim_ratio + gamma_focal_loss: + componentInputParameter: pipelinechannel--gamma_focal_loss + gradient_thresh: + componentInputParameter: pipelinechannel--gradient_thresh + large_category_dim: + componentInputParameter: pipelinechannel--large_category_dim + large_category_thresh: + componentInputParameter: pipelinechannel--large_category_thresh + learning_rate: + componentInputParameter: pipelinechannel--learning_rate + location: + componentInputParameter: pipelinechannel--location + loss_function_type: + componentInputParameter: pipelinechannel--loss_function_type + max_steps: + componentInputParameter: pipelinechannel--max_steps + max_train_secs: + componentInputParameter: pipelinechannel--max_train_secs + measurement_selection_type: + componentInputParameter: pipelinechannel--measurement_selection_type + num_decision_steps: + componentInputParameter: pipelinechannel--num_decision_steps + num_transformer_layers: + componentInputParameter: pipelinechannel--num_transformer_layers + num_transformer_layers_ratio: + componentInputParameter: pipelinechannel--num_transformer_layers_ratio + optimization_metric: + componentInputParameter: pipelinechannel--optimization_metric + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + relaxation_factor: + componentInputParameter: pipelinechannel--relaxation_factor + root_dir: + componentInputParameter: pipelinechannel--root_dir + seed: + componentInputParameter: pipelinechannel--seed + sparsity_loss_weight: + componentInputParameter: pipelinechannel--sparsity_loss_weight + target_column: + componentInputParameter: pipelinechannel--target_column + training_disk_spec: + taskOutputParameter: + outputParameterKey: training_disk_spec + producerTask: parse-worker-pool-specs-override + training_machine_spec: + taskOutputParameter: + outputParameterKey: training_machine_spec + producerTask: parse-worker-pool-specs-override + weight_column: + componentInputParameter: pipelinechannel--weight_column + yeo_johnson_transform: + componentInputParameter: pipelinechannel--yeo_johnson_transform + taskInfo: + name: tabnet-trainer + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--alpha_focal_loss: + parameterType: NUMBER_DOUBLE + pipelinechannel--batch_momentum: + parameterType: NUMBER_DOUBLE + pipelinechannel--batch_size: + parameterType: NUMBER_INTEGER + pipelinechannel--batch_size_ratio: + parameterType: NUMBER_DOUBLE + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--cache_data: + parameterType: STRING + pipelinechannel--class_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--dataset_level_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--dataset_level_transformations: + parameterType: LIST + pipelinechannel--decay_every: + parameterType: NUMBER_DOUBLE + pipelinechannel--decay_rate: + parameterType: NUMBER_DOUBLE + pipelinechannel--enable_profiler: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--eval_frequency_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--eval_steps: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_dim: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_dim_ratio: + parameterType: NUMBER_DOUBLE + pipelinechannel--feature_selection_algorithm: + parameterType: STRING + pipelinechannel--gamma_focal_loss: + parameterType: NUMBER_DOUBLE + pipelinechannel--gradient_thresh: + parameterType: NUMBER_DOUBLE + pipelinechannel--large_category_dim: + parameterType: NUMBER_INTEGER + pipelinechannel--large_category_thresh: + parameterType: NUMBER_INTEGER + pipelinechannel--learning_rate: + parameterType: NUMBER_DOUBLE + pipelinechannel--location: + parameterType: STRING + pipelinechannel--loss_function_type: + parameterType: STRING + pipelinechannel--materialized_examples_format: + parameterType: STRING + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--max_steps: + parameterType: NUMBER_INTEGER + pipelinechannel--max_train_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--measurement_selection_type: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--num_decision_steps: + parameterType: NUMBER_INTEGER + pipelinechannel--num_transformer_layers: + parameterType: NUMBER_INTEGER + pipelinechannel--num_transformer_layers_ratio: + parameterType: NUMBER_DOUBLE + pipelinechannel--optimization_metric: + parameterType: STRING + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--relaxation_factor: + parameterType: NUMBER_DOUBLE + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--run_feature_selection: + parameterType: BOOLEAN + pipelinechannel--seed: + parameterType: NUMBER_INTEGER + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--sparsity_loss_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--tf_auto_transform_features: + parameterType: STRUCT + pipelinechannel--tf_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--tf_transform_execution_engine: + parameterType: STRING + pipelinechannel--tf_transformations_path: + parameterType: STRING + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--worker_pool_specs_override: + parameterType: LIST + pipelinechannel--yeo_johnson_transform: + parameterType: BOOLEAN + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: An artifact of a model which to upload a new version to. Only + specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: "The unmanaged container model to be uploaded. The Model can\ + \ be passed from an upstream step or imported via a KFP `dsl.importer`.\n\ + :Examples:\n ::\n\n from kfp import dsl\n from google_cloud_pipeline_components.google_cloud_pipeline_components.types\ + \ import artifact_types\n\n importer_spec = dsl.importer(\n artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model',\n\ + \ artifact_class=artifact_types.UnmanagedContainerModel,\n metadata={\n\ + \ 'containerSpec': { 'imageUri':\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod'\n\ + \ }\n })" + isOptional: true + parameters: + description: + defaultValue: '' + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + isOptional: true + parameterType: STRING + display_name: + description: 'The display name of the Model. The name + + can be up to 128 characters long and can be consist of any UTF-8 + + characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model)' + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key spec for a Model. If set, this Model and all sub-resources of this + + Model will be secured by this key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + description: 'Metadata describing the Model''s + + input and output for explanation. Both `explanation_metadata` and `explanation_parameters` + must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata)' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters)' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your model. Label keys and values can be no longer than 64 + + characters (Unicode codepoints), can only contain lowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. See https://goo.gl/xmQnxf for more information and + + examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Optional location to upload this Model to. If + + not set, defaults to `us-central1`.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to upload this Model to. Defaults to the project in + which the PipelineJob is run. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: Artifact tracking the created Model. + parameters: + gcp_resources: + description: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) + which tracks the upload Model's long-running operation. + parameterType: STRING + comp-parse-worker-pool-specs-override: + executorLabel: exec-parse-worker-pool-specs-override + inputDefinitions: + parameters: + worker_pool_specs_override: + description: 'The list of dictionaries for overriding training + + and evaluation worker pool specs.' + parameterType: LIST + outputDefinitions: + parameters: + eval_machine_spec: + description: The eval machine spec. + parameterType: STRUCT + eval_replica_count: + description: The replica count for eval. + parameterType: NUMBER_INTEGER + training_disk_spec: + description: The training disk spec. + parameterType: STRUCT + training_machine_spec: + description: The training machine spec. + parameterType: STRUCT + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-tabnet-trainer: + executorLabel: exec-tabnet-trainer + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to a JSON file for baseline values. + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized validation split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized train split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Amount of time in seconds to run the trainer for. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to transform output. + parameters: + alpha_focal_loss: + defaultValue: 0.25 + description: 'Alpha value (balancing factor) in + + focal_loss function. Only used for classification.' + isOptional: true + parameterType: NUMBER_DOUBLE + batch_momentum: + defaultValue: 0.95 + description: Momentum in ghost batch normalization. + isOptional: true + parameterType: NUMBER_DOUBLE + batch_size: + defaultValue: 100.0 + description: Batch size for training. + isOptional: true + parameterType: NUMBER_INTEGER + batch_size_ratio: + defaultValue: 0.25 + description: 'The ratio of virtual batch size (size + + of the ghost batch normalization) to batch size.' + isOptional: true + parameterType: NUMBER_DOUBLE + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to + + ''auto'', caching is determined based on the dataset size.' + isOptional: true + parameterType: STRING + class_weight: + defaultValue: 1.0 + description: 'The class weight is used to computes a + + weighted cross entropy which is helpful in classify imbalanced dataset. + + Only used for classification.' + isOptional: true + parameterType: NUMBER_DOUBLE + decay_every: + defaultValue: 100.0 + description: 'Number of iterations for periodically + + applying learning rate decaying.' + isOptional: true + parameterType: NUMBER_DOUBLE + decay_rate: + defaultValue: 0.95 + description: Learning rate decaying. + isOptional: true + parameterType: NUMBER_DOUBLE + enable_profiler: + defaultValue: false + description: 'Enables profiling and saves a trace + + during evaluation.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and + + checkpointing will take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not + + specified or negative, it means run evaluation on the whole validation + + dataset. If set to 0, it means run evaluation for a fixed number of + + samples.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_dim: + defaultValue: 64.0 + description: 'Dimensionality of the hidden representation + + in feature transformation block.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_dim_ratio: + defaultValue: 0.5 + description: 'The ratio of output dimension + + (dimensionality of the outputs of each decision step) to feature + + dimension.' + isOptional: true + parameterType: NUMBER_DOUBLE + gamma_focal_loss: + defaultValue: 2.0 + description: 'Gamma value (modulating factor) for + + focal loss for focal loss. Only used for classification.' + isOptional: true + parameterType: NUMBER_DOUBLE + gradient_thresh: + defaultValue: 2000.0 + description: Threshold for the norm of gradients for clipping. + isOptional: true + parameterType: NUMBER_DOUBLE + large_category_dim: + defaultValue: 1.0 + description: 'Embedding dimension for categorical + + feature with large number of categories.' + isOptional: true + parameterType: NUMBER_INTEGER + large_category_thresh: + defaultValue: 300.0 + description: 'Threshold for number of categories + + to apply large_category_dim embedding dimension to.' + isOptional: true + parameterType: NUMBER_INTEGER + learning_rate: + description: The learning rate used by the linear optimizer. + parameterType: NUMBER_DOUBLE + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + loss_function_type: + defaultValue: default + description: 'Loss function type. Loss function in + + classification [cross_entropy, weighted_cross_entropy, focal_loss], + + default is cross_entropy. Loss function in regression: [rmse, mae, mse], + + default is mse.' + isOptional: true + parameterType: STRING + max_steps: + defaultValue: -1.0 + description: Number of steps to run the trainer for. + isOptional: true + parameterType: NUMBER_INTEGER + max_train_secs: + defaultValue: -1.0 + description: 'Amount of time in seconds to run the + + trainer for.' + isOptional: true + parameterType: NUMBER_INTEGER + measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: 'Which measurement to use + + if/when the service automatically selects the final measurement from + + previously reported intermediate measurements. One of "BEST_MEASUREMENT" + + or "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + num_decision_steps: + defaultValue: 6.0 + description: Number of sequential decision steps. + isOptional: true + parameterType: NUMBER_INTEGER + num_transformer_layers: + defaultValue: 4.0 + description: 'The number of transformer layers + + for each decision step. used only at one decision step and as it + + increases, more flexibility is provided to use a feature at multiple + + decision steps.' + isOptional: true + parameterType: NUMBER_INTEGER + num_transformer_layers_ratio: + defaultValue: 0.25 + description: 'The ratio of shared + + transformer layer to transformer layers.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_metric: + defaultValue: '' + description: 'Optimization metric used for + + `measurement_selection_type`. Default is "rmse" for regression and "auc" + + for classification.' + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to + + produce. "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + relaxation_factor: + defaultValue: 1.5 + description: 'Relaxation factor that promotes the + + reuse of each feature at different decision steps. When it is 1, a + + feature is enforced to be used only at one decision step and as it + + increases, more flexibility is provided to use a feature at multiple + + decision steps.' + isOptional: true + parameterType: NUMBER_DOUBLE + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + sparsity_loss_weight: + defaultValue: 1.0e-05 + description: 'Weight of the loss for sparsity + + regularization (increasing it will yield more sparse feature selection).' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + description: The target column name. + parameterType: STRING + training_disk_spec: + defaultValue: + boot_disk_size_gb: 100.0 + boot_disk_type: pd-ssd + description: The training disk spec. + isOptional: true + parameterType: STRUCT + training_machine_spec: + defaultValue: + machine_type: c2-standard-16 + description: 'The training machine + + spec. See https://cloud.google.com/compute/docs/machine-types for + + options.' + isOptional: true + parameterType: STRUCT + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + yeo_johnson_transform: + defaultValue: true + description: 'Enables trainable Yeo-Johnson + + power transform.' + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: The UnmanagedContainerModel artifact. + parameters: + gcp_resources: + description: Serialized gcp_resources proto tracking the custom training + job. + parameterType: STRING + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. +deploymentSpec: + executors: + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-infra-validator: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"pipeline_job\": + \"", "projects/{{$.inputs.parameters[''project'']}}/locations/{{$.inputs.parameters[''location'']}}/pipelineJobs/{{$.pipeline_job_uuid}}", + "\"", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.model.upload_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-parse-worker-pool-specs-override: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _parse_worker_pool_specs_override + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _parse_worker_pool_specs_override(\n worker_pool_specs_override:\ + \ list, # pylint:disable=g-bare-generic\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('training_machine_spec', dict), # pylint:disable=g-bare-generic\n\ + \ ('training_disk_spec', dict),\n ('eval_machine_spec', dict),\ + \ # pylint:disable=g-bare-generic\n ('eval_replica_count', int),\n\ + \ ],\n):\n \"\"\"Parses worker_pool_specs_override and returns training\ + \ and evaluation machine specifications.\n\n Args:\n worker_pool_specs_override:\ + \ The list of dictionaries for overriding training\n and evaluation\ + \ worker pool specs.\n\n Returns:\n training_machine_spec: The training\ + \ machine spec.\n training_disk_spec: The training disk spec.\n \ + \ eval_machine_spec: The eval machine spec.\n eval_replica_count:\ + \ The replica count for eval.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n training_machine_spec = {'machine_type': 'c2-standard-16'}\n training_disk_spec\ + \ = {'boot_disk_type': 'pd-ssd', 'boot_disk_size_gb': 100}\n eval_machine_spec\ + \ = {'machine_type': 'c2-standard-8'}\n eval_replica_count = 1\n\n if\ + \ worker_pool_specs_override:\n if len(worker_pool_specs_override) >=\ + \ 1 and isinstance(\n worker_pool_specs_override[0], dict\n ):\n\ + \ training_machine_spec = worker_pool_specs_override[0].get(\n \ + \ 'machine_spec', training_machine_spec\n )\n training_disk_spec\ + \ = worker_pool_specs_override[0].get(\n 'disk_spec', training_disk_spec\n\ + \ )\n if len(worker_pool_specs_override) == 4 and isinstance(\n\ + \ worker_pool_specs_override[3], dict\n ):\n eval_machine_spec\ + \ = worker_pool_specs_override[3].get(\n 'machine_spec', eval_machine_spec\n\ + \ )\n eval_replica_count = worker_pool_specs_override[3].get(\n\ + \ 'replica_count', eval_replica_count\n )\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'training_machine_spec',\n \ + \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ + \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ + \ eval_machine_spec,\n eval_replica_count,\n )\n\n" + image: python:3.7 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-tabnet-trainer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"tabnet-trainer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", "1", + "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", + ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/tabnet-training:20231002_0125", + "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", + "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", + "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_schema_path=", "{{$.inputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--job_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--training_data_path=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--validation_data_path=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--max_steps=", "{{$.inputs.parameters[''max_steps'']}}", "\", \"--max_train_secs=", + "{{$.inputs.parameters[''max_train_secs'']}}", "\", \"--learning_rate=", + "{{$.inputs.parameters[''learning_rate'']}}", "\", \"--large_category_dim=", + "{{$.inputs.parameters[''large_category_dim'']}}", "\", \"--large_category_thresh=", + "{{$.inputs.parameters[''large_category_thresh'']}}", "\", \"--yeo_johnson_transform=", + "{{$.inputs.parameters[''yeo_johnson_transform'']}}", "\", \"--feature_dim=", + "{{$.inputs.parameters[''feature_dim'']}}", "\", \"--feature_dim_ratio=", + "{{$.inputs.parameters[''feature_dim_ratio'']}}", "\", \"--num_decision_steps=", + "{{$.inputs.parameters[''num_decision_steps'']}}", "\", \"--relaxation_factor=", + "{{$.inputs.parameters[''relaxation_factor'']}}", "\", \"--decay_every=", + "{{$.inputs.parameters[''decay_every'']}}", "\", \"--decay_rate=", "{{$.inputs.parameters[''decay_rate'']}}", + "\", \"--gradient_thresh=", "{{$.inputs.parameters[''gradient_thresh'']}}", + "\", \"--sparsity_loss_weight=", "{{$.inputs.parameters[''sparsity_loss_weight'']}}", + "\", \"--batch_momentum=", "{{$.inputs.parameters[''batch_momentum'']}}", + "\", \"--batch_size_ratio=", "{{$.inputs.parameters[''batch_size_ratio'']}}", + "\", \"--num_transformer_layers=", "{{$.inputs.parameters[''num_transformer_layers'']}}", + "\", \"--num_transformer_layers_ratio=", "{{$.inputs.parameters[''num_transformer_layers_ratio'']}}", + "\", \"--class_weight=", "{{$.inputs.parameters[''class_weight'']}}", "\", + \"--loss_function_type=", "{{$.inputs.parameters[''loss_function_type'']}}", + "\", \"--alpha_focal_loss=", "{{$.inputs.parameters[''alpha_focal_loss'']}}", + "\", \"--gamma_focal_loss=", "{{$.inputs.parameters[''gamma_focal_loss'']}}", + "\", \"--enable_profiler=", "{{$.inputs.parameters[''enable_profiler'']}}", + "\", \"--cache_data=", "{{$.inputs.parameters[''cache_data'']}}", "\", \"--seed=", + "{{$.inputs.parameters[''seed'']}}", "\", \"--eval_steps=", "{{$.inputs.parameters[''eval_steps'']}}", + "\", \"--batch_size=", "{{$.inputs.parameters[''batch_size'']}}", "\", \"--measurement_selection_type=", + "{{$.inputs.parameters[''measurement_selection_type'']}}", "\", \"--optimization_metric=", + "{{$.inputs.parameters[''optimization_metric'']}}", "\", \"--eval_frequency_secs=", + "{{$.inputs.parameters[''eval_frequency_secs'']}}", "\", \"--generate_feature_importance=true\", + \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 +pipelineInfo: + description: 'Train a model using the Tabular Workflow for TabNet pipelines. + + TabNet uses sequential attention to choose which features to reason from at + + each decision step, promoting interpretability and more efficient learning.' + name: automl-tabular-tabnet-trainer +root: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--alpha_focal_loss: + componentInputParameter: alpha_focal_loss + pipelinechannel--batch_momentum: + componentInputParameter: batch_momentum + pipelinechannel--batch_size: + componentInputParameter: batch_size + pipelinechannel--batch_size_ratio: + componentInputParameter: batch_size_ratio + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + pipelinechannel--cache_data: + componentInputParameter: cache_data + pipelinechannel--class_weight: + componentInputParameter: class_weight + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + pipelinechannel--dataset_level_transformations: + componentInputParameter: dataset_level_transformations + pipelinechannel--decay_every: + componentInputParameter: decay_every + pipelinechannel--decay_rate: + componentInputParameter: decay_rate + pipelinechannel--enable_profiler: + componentInputParameter: enable_profiler + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--eval_frequency_secs: + componentInputParameter: eval_frequency_secs + pipelinechannel--eval_steps: + componentInputParameter: eval_steps + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--feature_dim: + componentInputParameter: feature_dim + pipelinechannel--feature_dim_ratio: + componentInputParameter: feature_dim_ratio + pipelinechannel--feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + pipelinechannel--gamma_focal_loss: + componentInputParameter: gamma_focal_loss + pipelinechannel--gradient_thresh: + componentInputParameter: gradient_thresh + pipelinechannel--large_category_dim: + componentInputParameter: large_category_dim + pipelinechannel--large_category_thresh: + componentInputParameter: large_category_thresh + pipelinechannel--learning_rate: + componentInputParameter: learning_rate + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--loss_function_type: + componentInputParameter: loss_function_type + pipelinechannel--materialized_examples_format: + componentInputParameter: materialized_examples_format + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--max_steps: + componentInputParameter: max_steps + pipelinechannel--max_train_secs: + componentInputParameter: max_train_secs + pipelinechannel--measurement_selection_type: + componentInputParameter: measurement_selection_type + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--num_decision_steps: + componentInputParameter: num_decision_steps + pipelinechannel--num_transformer_layers: + componentInputParameter: num_transformer_layers + pipelinechannel--num_transformer_layers_ratio: + componentInputParameter: num_transformer_layers_ratio + pipelinechannel--optimization_metric: + componentInputParameter: optimization_metric + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--prediction_type: + componentInputParameter: prediction_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--relaxation_factor: + componentInputParameter: relaxation_factor + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--run_feature_selection: + componentInputParameter: run_feature_selection + pipelinechannel--seed: + componentInputParameter: seed + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--sparsity_loss_weight: + componentInputParameter: sparsity_loss_weight + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + pipelinechannel--tf_custom_transformation_definitions: + componentInputParameter: tf_custom_transformation_definitions + pipelinechannel--tf_transform_execution_engine: + componentInputParameter: tf_transform_execution_engine + pipelinechannel--tf_transformations_path: + componentInputParameter: tf_transformations_path + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--worker_pool_specs_override: + componentInputParameter: worker_pool_specs_override + pipelinechannel--yeo_johnson_transform: + componentInputParameter: yeo_johnson_transform + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Parent model if this model is uploaded as a version. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + alpha_focal_loss: + defaultValue: 0.25 + description: 'Alpha value (balancing factor) in focal_loss function. + + Only used for classification.' + isOptional: true + parameterType: NUMBER_DOUBLE + batch_momentum: + defaultValue: 0.95 + description: Momentum in ghost batch normalization. + isOptional: true + parameterType: NUMBER_DOUBLE + batch_size: + defaultValue: 100.0 + description: Batch size for training. + isOptional: true + parameterType: NUMBER_INTEGER + batch_size_ratio: + defaultValue: 0.25 + description: 'The ratio of virtual batch size (size of the ghost batch + + normalization) to batch size.' + isOptional: true + parameterType: NUMBER_DOUBLE + bigquery_staging_full_dataset_id: + defaultValue: '' + description: Staging directory for BigQuery tables. + isOptional: true + parameterType: STRING + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to ''auto'', caching is + + determined based on the dataset size.' + isOptional: true + parameterType: STRING + class_weight: + defaultValue: 1.0 + description: 'The class weight is used to computes a weighted cross entropy + + which is helpful in classify imbalanced dataset. Only used for + + classification.' + isOptional: true + parameterType: NUMBER_DOUBLE + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + description: 'Dataset-level custom + + transformation definitions in string format.' + isOptional: true + parameterType: LIST + dataset_level_transformations: + description: 'Dataset-level transformation configuration in + + string format.' + isOptional: true + parameterType: LIST + decay_every: + defaultValue: 100.0 + description: 'Number of iterations for periodically applying learning rate + + decaying.' + isOptional: true + parameterType: NUMBER_DOUBLE + decay_rate: + defaultValue: 0.95 + description: Learning rate decaying. + isOptional: true + parameterType: NUMBER_DOUBLE + enable_profiler: + defaultValue: false + description: Enables profiling and saves a trace during evaluation. + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and checkpointing will + + take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not specified or + + negative, it means run evaluation on the whole validation dataset. If set + + to 0, it means run evaluation for a fixed number of samples.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_dim: + defaultValue: 64.0 + description: 'Dimensionality of the hidden representation in feature + + transformation block.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_dim_ratio: + defaultValue: 0.5 + description: 'The ratio of output dimension (dimensionality of the + + outputs of each decision step) to feature dimension.' + isOptional: true + parameterType: NUMBER_DOUBLE + feature_selection_algorithm: + defaultValue: AMI + description: Feature selection algorithm. + isOptional: true + parameterType: STRING + gamma_focal_loss: + defaultValue: 2.0 + description: 'Gamma value (modulating factor) for focal loss for focal + + loss. Only used for classification.' + isOptional: true + parameterType: NUMBER_DOUBLE + gradient_thresh: + defaultValue: 2000.0 + description: Threshold for the norm of gradients for clipping. + isOptional: true + parameterType: NUMBER_DOUBLE + large_category_dim: + defaultValue: 1.0 + description: 'Embedding dimension for categorical feature with large + + number of categories.' + isOptional: true + parameterType: NUMBER_INTEGER + large_category_thresh: + defaultValue: 300.0 + description: 'Threshold for number of categories to apply + + large_category_dim embedding dimension to.' + isOptional: true + parameterType: NUMBER_INTEGER + learning_rate: + description: The learning rate used by the linear optimizer. + parameterType: NUMBER_DOUBLE + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + loss_function_type: + defaultValue: default + description: 'Loss function type. Loss function in classification + + [cross_entropy, weighted_cross_entropy, focal_loss], default is + + cross_entropy. Loss function in regression: [rmse, mae, mse], default is + + mse.' + isOptional: true + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: The format for the materialized examples. + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: -1.0 + description: Maximum number of features to select. + isOptional: true + parameterType: NUMBER_INTEGER + max_steps: + defaultValue: -1.0 + description: Number of steps to run the trainer for. + isOptional: true + parameterType: NUMBER_INTEGER + max_train_secs: + defaultValue: -1.0 + description: Amount of time in seconds to run the trainer for. + isOptional: true + parameterType: NUMBER_INTEGER + measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: 'Which measurement to use if/when the service + + automatically selects the final measurement from previously reported + + intermediate measurements. One of "BEST_MEASUREMENT" or + + "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + num_decision_steps: + defaultValue: 6.0 + description: Number of sequential decision steps. + isOptional: true + parameterType: NUMBER_INTEGER + num_transformer_layers: + defaultValue: 4.0 + description: 'The number of transformer layers for each decision + + step. used only at one decision step and as it increases, more flexibility + + is provided to use a feature at multiple decision steps.' + isOptional: true + parameterType: NUMBER_INTEGER + num_transformer_layers_ratio: + defaultValue: 0.25 + description: 'The ratio of shared transformer layer to + + transformer layers.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_metric: + defaultValue: '' + description: 'Optimization metric used for + + `measurement_selection_type`. Default is "rmse" for regression and "auc" + + for classification.' + isOptional: true + parameterType: STRING + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to produce. + + "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + relaxation_factor: + defaultValue: 1.5 + description: 'Relaxation factor that promotes the reuse of each feature + + at different decision steps. When it is 1, a feature is enforced to be + + used only at one decision step and as it increases, more flexibility is + + provided to use a feature at multiple decision steps.' + isOptional: true + parameterType: NUMBER_DOUBLE + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: Whether to enable feature selection. + isOptional: true + parameterType: BOOLEAN + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + sparsity_loss_weight: + defaultValue: 1.0e-05 + description: 'Weight of the loss for sparsity regularization + + (increasing it will yield more sparse feature selection).' + isOptional: true + parameterType: NUMBER_DOUBLE + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: Test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + description: List of auto transform features. + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + description: 'TF custom transformation definitions + + in string format.' + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: '' + description: 'Execution engine to run TF-based + + transformations. Currently supports "dataflow" or "bigquery"' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: Path to TF transformation configuration. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + validation_fraction: + defaultValue: -1.0 + description: Validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + worker_pool_specs_override: + description: 'The dictionary for overriding training and + + evaluation worker pool specs. The dictionary should be of format + + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172.' + isOptional: true + parameterType: LIST + yeo_johnson_transform: + defaultValue: true + description: Enables trainable Yeo-Johnson power transform. + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/utils.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/utils.py new file mode 100644 index 0000000000..97e6b370a1 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/utils.py @@ -0,0 +1,3453 @@ +"""Util functions for AutoML Tabular pipeline.""" + +import json +import os +import pathlib +from typing import Any, Dict, List, Optional, Tuple, Union +import uuid +import warnings + +_DEFAULT_NUM_PARALLEL_TRAILS = 35 +_DEFAULT_STAGE_2_NUM_SELECTED_TRAILS = 5 +_NUM_FOLDS = 5 +_DISTILL_TOTAL_TRIALS = 100 +_EVALUATION_BATCH_PREDICT_MACHINE_TYPE = 'n1-highmem-8' +_EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT = 20 +_EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT = 20 +_EVALUATION_BATCH_EXPLAIN_MACHINE_TYPE = 'n1-highmem-8' +_EVALUATION_BATCH_EXPLAIN_STARTING_REPLICA_COUNT = 10 +_EVALUATION_BATCH_EXPLAIN_MAX_REPLICA_COUNT = 10 +_EVALUATION_DATAFLOW_MACHINE_TYPE = 'n1-standard-4' +_EVALUATION_DATAFLOW_STARTING_NUM_WORKERS = 10 +_EVALUATION_DATAFLOW_MAX_NUM_WORKERS = 100 +_EVALUATION_DATAFLOW_DISK_SIZE_GB = 50 +_FEATURE_SELECTION_EXECUTION_ENGINE_BIGQUERY = 'bigquery' + +# Needed because we reference the AutoML Tabular V1 pipeline. +_GCPC_STAGING_PATH = pathlib.Path( + __file__ +).parent.parent.parent.parent.resolve() +_GCPC_GA_TABULAR_PATH = str(_GCPC_STAGING_PATH / 'v1' / 'automl' / 'tabular') + + +def _update_parameters( + parameter_values: Dict[str, Any], new_params: Dict[str, Any] +): + parameter_values.update( + {param: value for param, value in new_params.items() if value is not None} + ) + + +def _generate_model_display_name() -> str: + """Automatically generates a model_display_name. + + Returns: + model_display_name. + """ + return f'tabular-workflow-model-{uuid.uuid4()}' + + +def _get_default_pipeline_params( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + transformations: str, + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: Optional[int] = None, + stage_2_num_parallel_trials: Optional[int] = None, + stage_2_num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[float] = None, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + optimization_objective_recall_value: Optional[float] = None, + optimization_objective_precision_value: Optional[float] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: Optional[str] = None, + stats_and_example_gen_dataflow_max_num_workers: Optional[int] = None, + stats_and_example_gen_dataflow_disk_size_gb: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: Optional[str] = None, + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: Optional[str] = None, + max_selected_features: Optional[int] = None, + apply_feature_selection_tuning: bool = False, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_batch_explain_machine_type: Optional[str] = None, + evaluation_batch_explain_starting_replica_count: Optional[int] = None, + evaluation_batch_explain_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, + run_distillation: bool = False, + distill_batch_predict_machine_type: Optional[str] = None, + distill_batch_predict_starting_replica_count: Optional[int] = None, + distill_batch_predict_max_replica_count: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + quantiles: Optional[List[float]] = None, + enable_probabilistic_inference: bool = False, + num_selected_features: Optional[int] = None, + model_display_name: str = '', + model_description: str = '', + enable_fte: bool = False, +) -> Dict[str, Any]: + """Get the AutoML Tabular v1 default training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The path to a GCS file containing the transformations to + apply. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + stratified_split_key: The stratified_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + weight_column: The weight column name. + study_spec_parameters_override: The list for overriding study spec. The list + should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + max_selected_features: number of features to select for training, + apply_feature_selection_tuning: tuning feature selection rate if true. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_batch_explain_machine_type: The prediction server machine type + for batch explain components during evaluation. + evaluation_batch_explain_starting_replica_count: The initial number of + prediction server for batch explain components during evaluation. + evaluation_batch_explain_max_replica_count: The max number of prediction + server for batch explain components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + run_distillation: Whether to run distill in the training pipeline. + distill_batch_predict_machine_type: The prediction server machine type for + batch predict component in the model distillation. + distill_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict component in the model distillation. + distill_batch_predict_max_replica_count: The max number of prediction server + for batch predict component in the model distillation. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + quantiles: Quantiles to use for probabilistic inference. Up to 5 quantiles + are allowed of values between 0 and 1, exclusive. Represents the quantiles + to use for that objective. Quantiles must be unique. + enable_probabilistic_inference: If probabilistic inference is enabled, the + model will fit a distribution that captures the uncertainty of a + prediction. At inference time, the predictive distribution is used to make + a point prediction that minimizes the optimization objective. For example, + the mean of a predictive distribution is the point prediction that + minimizes RMSE loss. If quantiles are specified, then the quantiles of the + distribution are also returned. + num_selected_features: Number of selected features for feature selection, + defaults to None, in which case all features are used. If specified, + enable_probabilistic_inference and run_distillation cannot be enabled. + model_display_name: The display name of the uploaded Vertex model. + model_description: The description for the uploaded model. + enable_fte: Whether to enable the Feature Transform Engine. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + if not study_spec_parameters_override: + study_spec_parameters_override = [] + if not stage_1_tuner_worker_pool_specs_override: + stage_1_tuner_worker_pool_specs_override = [] + if not cv_trainer_worker_pool_specs_override: + cv_trainer_worker_pool_specs_override = [] + if not quantiles: + quantiles = [] + + parameter_values = {} + parameters = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'prediction_type': prediction_type, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'predefined_split_key': predefined_split_key, + 'timestamp_split_key': timestamp_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'optimization_objective': optimization_objective, + 'train_budget_milli_node_hours': train_budget_milli_node_hours, + 'stage_1_num_parallel_trials': stage_1_num_parallel_trials, + 'stage_2_num_parallel_trials': stage_2_num_parallel_trials, + 'stage_2_num_selected_trials': stage_2_num_selected_trials, + 'weight_column': weight_column, + 'optimization_objective_recall_value': ( + optimization_objective_recall_value + ), + 'optimization_objective_precision_value': ( + optimization_objective_precision_value + ), + 'study_spec_parameters_override': study_spec_parameters_override, + 'stage_1_tuner_worker_pool_specs_override': ( + stage_1_tuner_worker_pool_specs_override + ), + 'cv_trainer_worker_pool_specs_override': ( + cv_trainer_worker_pool_specs_override + ), + 'export_additional_model_without_custom_ops': ( + export_additional_model_without_custom_ops + ), + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'dataflow_service_account': dataflow_service_account, + 'encryption_spec_key_name': encryption_spec_key_name, + 'max_selected_features': max_selected_features, + 'stage_1_tuning_result_artifact_uri': stage_1_tuning_result_artifact_uri, + 'quantiles': quantiles, + 'enable_probabilistic_inference': enable_probabilistic_inference, + 'model_display_name': model_display_name, + 'model_description': model_description, + } + parameter_values.update( + {param: value for param, value in parameters.items() if value is not None} + ) + + if run_evaluation: + eval_parameters = { + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_batch_explain_machine_type': ( + evaluation_batch_explain_machine_type + ), + 'evaluation_batch_explain_starting_replica_count': ( + evaluation_batch_explain_starting_replica_count + ), + 'evaluation_batch_explain_max_replica_count': ( + evaluation_batch_explain_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'run_evaluation': run_evaluation, + } + parameter_values.update( + { + param: value + for param, value in eval_parameters.items() + if value is not None + } + ) + + if run_distillation: + distillation_parameters = { + 'distill_batch_predict_machine_type': ( + distill_batch_predict_machine_type + ), + 'distill_batch_predict_starting_replica_count': ( + distill_batch_predict_starting_replica_count + ), + 'distill_batch_predict_max_replica_count': ( + distill_batch_predict_max_replica_count + ), + 'run_distillation': run_distillation, + } + parameter_values.update( + { + param: value + for param, value in distillation_parameters.items() + if value is not None + } + ) + + # V1 pipeline + if not enable_fte: + if not additional_experiments: + additional_experiments = {} + + parameters = { + 'transformations': transformations, + 'stats_and_example_gen_dataflow_machine_type': ( + stats_and_example_gen_dataflow_machine_type + ), + 'stats_and_example_gen_dataflow_max_num_workers': ( + stats_and_example_gen_dataflow_max_num_workers + ), + 'stats_and_example_gen_dataflow_disk_size_gb': ( + stats_and_example_gen_dataflow_disk_size_gb + ), + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': ( + transform_dataflow_max_num_workers + ), + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'additional_experiments': additional_experiments, + } + parameter_values.update( + { + param: value + for param, value in parameters.items() + if value is not None + } + ) + + if apply_feature_selection_tuning: + parameter_values.update({ + 'apply_feature_selection_tuning': apply_feature_selection_tuning, + }) + + # V2 pipeline (with FTE) + else: + parameters = { + 'num_selected_features': num_selected_features, + 'dataset_level_custom_transformation_definitions': [], + 'dataset_level_transformations': [], + 'tf_auto_transform_features': {}, + 'tf_custom_transformation_definitions': [], + 'legacy_transformations_path': transformations, + 'feature_transform_engine_dataflow_machine_type': ( + transform_dataflow_machine_type + ), + 'feature_transform_engine_dataflow_max_num_workers': ( + transform_dataflow_max_num_workers + ), + 'feature_transform_engine_dataflow_disk_size_gb': ( + transform_dataflow_disk_size_gb + ), + } + parameter_values.update( + { + param: value + for param, value in parameters.items() + if value is not None + } + ) + + return parameter_values + + +def get_automl_tabular_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + transformations: str, + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: Optional[int] = None, + stage_2_num_parallel_trials: Optional[int] = None, + stage_2_num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + optimization_objective_recall_value: Optional[float] = None, + optimization_objective_precision_value: Optional[float] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: Optional[str] = None, + stats_and_example_gen_dataflow_max_num_workers: Optional[int] = None, + stats_and_example_gen_dataflow_disk_size_gb: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: Optional[str] = None, + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: Optional[str] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_batch_explain_machine_type: Optional[str] = None, + evaluation_batch_explain_starting_replica_count: Optional[int] = None, + evaluation_batch_explain_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, + run_distillation: bool = False, + distill_batch_predict_machine_type: Optional[str] = None, + distill_batch_predict_starting_replica_count: Optional[int] = None, + distill_batch_predict_max_replica_count: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + quantiles: Optional[List[float]] = None, + enable_probabilistic_inference: bool = False, + num_selected_features: Optional[int] = None, + model_display_name: str = '', + model_description: str = '', + enable_fte: bool = False, +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular v1 default training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The path to a GCS file containing the transformations to + apply. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + stratified_split_key: The stratified_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + weight_column: The weight column name. + study_spec_parameters_override: The list for overriding study spec. The list + should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_batch_explain_machine_type: The prediction server machine type + for batch explain components during evaluation. + evaluation_batch_explain_starting_replica_count: The initial number of + prediction server for batch explain components during evaluation. + evaluation_batch_explain_max_replica_count: The max number of prediction + server for batch explain components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + run_distillation: Whether to run distill in the training pipeline. + distill_batch_predict_machine_type: The prediction server machine type for + batch predict component in the model distillation. + distill_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict component in the model distillation. + distill_batch_predict_max_replica_count: The max number of prediction server + for batch predict component in the model distillation. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + quantiles: Quantiles to use for probabilistic inference. Up to 5 quantiles + are allowed of values between 0 and 1, exclusive. Represents the quantiles + to use for that objective. Quantiles must be unique. + enable_probabilistic_inference: If probabilistic inference is enabled, the + model will fit a distribution that captures the uncertainty of a + prediction. At inference time, the predictive distribution is used to make + a point prediction that minimizes the optimization objective. For example, + the mean of a predictive distribution is the point prediction that + minimizes RMSE loss. If quantiles are specified, then the quantiles of the + distribution are also returned. + num_selected_features: Number of selected features for feature selection, + defaults to None, in which case all features are used. + model_display_name: The display name of the uploaded Vertex model. + model_description: The description for the uploaded model. + enable_fte: Whether to enable the Feature Transform Engine. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = _get_default_pipeline_params( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + prediction_type=prediction_type, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + stage_2_num_selected_trials=stage_2_num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + timestamp_split_key=timestamp_split_key, + stratified_split_key=stratified_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + study_spec_parameters_override=study_spec_parameters_override, + optimization_objective_recall_value=optimization_objective_recall_value, + optimization_objective_precision_value=optimization_objective_precision_value, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + cv_trainer_worker_pool_specs_override=cv_trainer_worker_pool_specs_override, + export_additional_model_without_custom_ops=export_additional_model_without_custom_ops, + stats_and_example_gen_dataflow_machine_type=stats_and_example_gen_dataflow_machine_type, + stats_and_example_gen_dataflow_max_num_workers=stats_and_example_gen_dataflow_max_num_workers, + stats_and_example_gen_dataflow_disk_size_gb=stats_and_example_gen_dataflow_disk_size_gb, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + additional_experiments=additional_experiments, + dataflow_service_account=dataflow_service_account, + run_evaluation=run_evaluation, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_batch_explain_machine_type=evaluation_batch_explain_machine_type, + evaluation_batch_explain_starting_replica_count=evaluation_batch_explain_starting_replica_count, + evaluation_batch_explain_max_replica_count=evaluation_batch_explain_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + run_distillation=run_distillation, + distill_batch_predict_machine_type=distill_batch_predict_machine_type, + distill_batch_predict_starting_replica_count=distill_batch_predict_starting_replica_count, + distill_batch_predict_max_replica_count=distill_batch_predict_max_replica_count, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + quantiles=quantiles, + enable_probabilistic_inference=enable_probabilistic_inference, + num_selected_features=num_selected_features, + model_display_name=model_display_name, + model_description=model_description, + enable_fte=enable_fte, + ) + + # V1 pipeline without FTE + if not enable_fte: + pipeline_definition_path = os.path.join( + _GCPC_GA_TABULAR_PATH, 'automl_tabular_pipeline.yaml' + ) + + # V2 pipeline with FTE + else: + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'automl_tabular_v2_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_automl_tabular_feature_selection_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + transformations: str, + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: Optional[int] = None, + stage_2_num_parallel_trials: Optional[int] = None, + stage_2_num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + optimization_objective_recall_value: Optional[float] = None, + optimization_objective_precision_value: Optional[float] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: Optional[str] = None, + stats_and_example_gen_dataflow_max_num_workers: Optional[int] = None, + stats_and_example_gen_dataflow_disk_size_gb: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: Optional[str] = None, + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: Optional[str] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_batch_explain_machine_type: Optional[str] = None, + evaluation_batch_explain_starting_replica_count: Optional[int] = None, + evaluation_batch_explain_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, + max_selected_features: int = 1000, + apply_feature_selection_tuning: bool = False, + run_distillation: bool = False, + distill_batch_predict_machine_type: Optional[str] = None, + distill_batch_predict_starting_replica_count: Optional[int] = None, + distill_batch_predict_max_replica_count: Optional[int] = None, + model_display_name: str = '', + model_description: str = '', +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular v1 default training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The path to a GCS file containing the transformations to + apply. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + stratified_split_key: The stratified_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + weight_column: The weight column name. + study_spec_parameters_override: The list for overriding study spec. The list + should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_batch_explain_machine_type: The prediction server machine type + for batch explain components during evaluation. + evaluation_batch_explain_starting_replica_count: The initial number of + prediction server for batch explain components during evaluation. + evaluation_batch_explain_max_replica_count: The max number of prediction + server for batch explain components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + max_selected_features: number of features to select for training, + apply_feature_selection_tuning: tuning feature selection rate if true. + run_distillation: Whether to run distill in the training pipeline. + distill_batch_predict_machine_type: The prediction server machine type for + batch predict component in the model distillation. + distill_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict component in the model distillation. + distill_batch_predict_max_replica_count: The max number of prediction server + for batch predict component in the model distillation. + model_display_name: The display name of the uploaded Vertex model. + model_description: The description for the uploaded model. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + model_display_name = ( + model_display_name + if model_display_name + else _generate_model_display_name() + ) + + parameter_values = _get_default_pipeline_params( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + prediction_type=prediction_type, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + stage_2_num_selected_trials=stage_2_num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + timestamp_split_key=timestamp_split_key, + stratified_split_key=stratified_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + study_spec_parameters_override=study_spec_parameters_override, + optimization_objective_recall_value=optimization_objective_recall_value, + optimization_objective_precision_value=optimization_objective_precision_value, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + cv_trainer_worker_pool_specs_override=cv_trainer_worker_pool_specs_override, + export_additional_model_without_custom_ops=export_additional_model_without_custom_ops, + stats_and_example_gen_dataflow_machine_type=stats_and_example_gen_dataflow_machine_type, + stats_and_example_gen_dataflow_max_num_workers=stats_and_example_gen_dataflow_max_num_workers, + stats_and_example_gen_dataflow_disk_size_gb=stats_and_example_gen_dataflow_disk_size_gb, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + additional_experiments=additional_experiments, + dataflow_service_account=dataflow_service_account, + max_selected_features=max_selected_features, + apply_feature_selection_tuning=apply_feature_selection_tuning, + run_evaluation=run_evaluation, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_batch_explain_machine_type=evaluation_batch_explain_machine_type, + evaluation_batch_explain_starting_replica_count=evaluation_batch_explain_starting_replica_count, + evaluation_batch_explain_max_replica_count=evaluation_batch_explain_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + run_distillation=run_distillation, + distill_batch_predict_machine_type=distill_batch_predict_machine_type, + distill_batch_predict_starting_replica_count=distill_batch_predict_starting_replica_count, + distill_batch_predict_max_replica_count=distill_batch_predict_max_replica_count, + model_display_name=model_display_name, + model_description=model_description, + ) + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'automl_tabular_feature_selection_pipeline.yaml', + ) + return pipeline_definition_path, parameter_values + + +def input_dictionary_to_parameter(input_dict: Optional[Dict[str, Any]]) -> str: + """Convert json input dict to encoded parameter string. + + This function is required due to the limitation on YAML component definition + that YAML definition does not have a keyword for apply quote escape, so the + JSON argument's quote must be manually escaped using this function. + + Args: + input_dict: The input json dictionary. + + Returns: + The encoded string used for parameter. + """ + if not input_dict: + return '' + out = json.dumps(json.dumps(input_dict)) + return out[1:-1] # remove the outside quotes, e.g., "foo" -> foo + + +def get_skip_architecture_search_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + transformations: str, + train_budget_milli_node_hours: float, + stage_1_tuning_result_artifact_uri: str, + stage_2_num_parallel_trials: Optional[int] = None, + stage_2_num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + optimization_objective_recall_value: Optional[float] = None, + optimization_objective_precision_value: Optional[float] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: Optional[str] = None, + stats_and_example_gen_dataflow_max_num_workers: Optional[int] = None, + stats_and_example_gen_dataflow_disk_size_gb: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: Optional[str] = None, + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: Optional[str] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_batch_explain_machine_type: Optional[str] = None, + evaluation_batch_explain_starting_replica_count: Optional[int] = None, + evaluation_batch_explain_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular training pipeline that skips architecture search. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The transformations to apply. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + stratified_split_key: The stratified_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + weight_column: The weight column name. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_batch_explain_machine_type: The prediction server machine type + for batch explain components during evaluation. + evaluation_batch_explain_starting_replica_count: The initial number of + prediction server for batch explain components during evaluation. + evaluation_batch_explain_max_replica_count: The max number of prediction + server for batch explain components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + + return get_automl_tabular_pipeline_and_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + prediction_type=prediction_type, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + stage_1_num_parallel_trials=None, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + stage_2_num_selected_trials=stage_2_num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + timestamp_split_key=timestamp_split_key, + stratified_split_key=stratified_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + study_spec_parameters_override=[], + optimization_objective_recall_value=optimization_objective_recall_value, + optimization_objective_precision_value=optimization_objective_precision_value, + stage_1_tuner_worker_pool_specs_override={}, + cv_trainer_worker_pool_specs_override=cv_trainer_worker_pool_specs_override, + export_additional_model_without_custom_ops=export_additional_model_without_custom_ops, + stats_and_example_gen_dataflow_machine_type=stats_and_example_gen_dataflow_machine_type, + stats_and_example_gen_dataflow_max_num_workers=stats_and_example_gen_dataflow_max_num_workers, + stats_and_example_gen_dataflow_disk_size_gb=stats_and_example_gen_dataflow_disk_size_gb, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + additional_experiments=additional_experiments, + dataflow_service_account=dataflow_service_account, + run_evaluation=run_evaluation, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_batch_explain_machine_type=evaluation_batch_explain_machine_type, + evaluation_batch_explain_starting_replica_count=evaluation_batch_explain_starting_replica_count, + evaluation_batch_explain_max_replica_count=evaluation_batch_explain_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + run_distillation=None, + distill_batch_predict_machine_type=None, + distill_batch_predict_starting_replica_count=None, + distill_batch_predict_max_replica_count=None, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + quantiles=[], + enable_probabilistic_inference=False, + ) + + +def get_wide_and_deep_trainer_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + learning_rate: float, + dnn_learning_rate: float, + transform_config: Optional[str] = None, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + run_feature_selection: bool = False, + feature_selection_algorithm: Optional[str] = None, + materialized_examples_format: Optional[str] = None, + max_selected_features: Optional[int] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_transform_execution_engine: Optional[str] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + tf_custom_transformation_definitions: Optional[List[Dict[str, Any]]] = None, + tf_transformations_path: Optional[str] = None, + optimizer_type: str = 'adam', + max_steps: int = -1, + max_train_secs: int = -1, + l1_regularization_strength: float = 0, + l2_regularization_strength: float = 0, + l2_shrinkage_regularization_strength: float = 0, + beta_1: float = 0.9, + beta_2: float = 0.999, + hidden_units: str = '30,30,30', + use_wide: bool = True, + embed_categories: bool = True, + dnn_dropout: float = 0, + dnn_optimizer_type: str = 'adam', + dnn_l1_regularization_strength: float = 0, + dnn_l2_regularization_strength: float = 0, + dnn_l2_shrinkage_regularization_strength: float = 0, + dnn_beta_1: float = 0.9, + dnn_beta_2: float = 0.999, + enable_profiler: bool = False, + cache_data: str = 'auto', + seed: int = 1, + eval_steps: int = 0, + batch_size: int = 100, + measurement_selection_type: Optional[str] = None, + optimization_metric: Optional[str] = None, + eval_frequency_secs: int = 600, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + weight_column: str = '', + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + worker_pool_specs_override: Optional[Dict[str, Any]] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: str = _EVALUATION_BATCH_PREDICT_MACHINE_TYPE, + evaluation_batch_predict_starting_replica_count: int = _EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT, + evaluation_batch_predict_max_replica_count: int = _EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT, + evaluation_dataflow_machine_type: str = _EVALUATION_DATAFLOW_MACHINE_TYPE, + evaluation_dataflow_starting_num_workers: int = _EVALUATION_DATAFLOW_STARTING_NUM_WORKERS, + evaluation_dataflow_max_num_workers: int = _EVALUATION_DATAFLOW_MAX_NUM_WORKERS, + evaluation_dataflow_disk_size_gb: int = _EVALUATION_DATAFLOW_DISK_SIZE_GB, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +) -> Tuple[str, Dict[str, Any]]: + """Get the Wide & Deep training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + 'classification' or 'regression'. + learning_rate: The learning rate used by the linear optimizer. + dnn_learning_rate: The learning rate for training the deep part of the + model. + transform_config: Path to v1 TF transformation configuration. + dataset_level_custom_transformation_definitions: Dataset-level custom + transformation definitions in string format. + dataset_level_transformations: Dataset-level transformation configuration in + string format. + run_feature_selection: Whether to enable feature selection. + feature_selection_algorithm: Feature selection algorithm. + materialized_examples_format: The format for the materialized examples. + max_selected_features: Maximum number of features to select. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + training_fraction: Training fraction. + validation_fraction: Validation fraction. + test_fraction: Test fraction. + tf_transform_execution_engine: The execution engine used to execute TF-based + transformations. + tf_auto_transform_features: List of auto transform features in the + comma-separated string format. + tf_custom_transformation_definitions: TF custom transformation definitions + in string format. + tf_transformations_path: Path to TF transformation configuration. + optimizer_type: The type of optimizer to use. Choices are "adam", "ftrl" and + "sgd" for the Adam, FTRL, and Gradient Descent Optimizers, respectively. + max_steps: Number of steps to run the trainer for. + max_train_secs: Amount of time in seconds to run the trainer for. + l1_regularization_strength: L1 regularization strength for + optimizer_type="ftrl". + l2_regularization_strength: L2 regularization strength for + optimizer_type="ftrl". + l2_shrinkage_regularization_strength: L2 shrinkage regularization strength + for optimizer_type="ftrl". + beta_1: Beta 1 value for optimizer_type="adam". + beta_2: Beta 2 value for optimizer_type="adam". + hidden_units: Hidden layer sizes to use for DNN feature columns, provided in + comma-separated layers. + use_wide: If set to true, the categorical columns will be used in the wide + part of the DNN model. + embed_categories: If set to true, the categorical columns will be used + embedded and used in the deep part of the model. Embedding size is the + square root of the column cardinality. + dnn_dropout: The probability we will drop out a given coordinate. + dnn_optimizer_type: The type of optimizer to use for the deep part of the + model. Choices are "adam", "ftrl" and "sgd". for the Adam, FTRL, and + Gradient Descent Optimizers, respectively. + dnn_l1_regularization_strength: L1 regularization strength for + dnn_optimizer_type="ftrl". + dnn_l2_regularization_strength: L2 regularization strength for + dnn_optimizer_type="ftrl". + dnn_l2_shrinkage_regularization_strength: L2 shrinkage regularization + strength for dnn_optimizer_type="ftrl". + dnn_beta_1: Beta 1 value for dnn_optimizer_type="adam". + dnn_beta_2: Beta 2 value for dnn_optimizer_type="adam". + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is + determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or + negative, it means run evaluation on the whole validation dataset. If set + to 0, it means run evaluation for a fixed number of samples. + batch_size: Batch size for training. + measurement_selection_type: Which measurement to use if/when the service + automatically selects the final measurement from previously reported + intermediate measurements. One of "BEST_MEASUREMENT" or + "LAST_MEASUREMENT". + optimization_metric: Optimization metric used for + `measurement_selection_type`. Default is "rmse" for regression and "auc" + for classification. + eval_frequency_secs: Frequency at which evaluation and checkpointing will + take place. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + bigquery_staging_full_dataset_id: The BigQuery staging full dataset id for + storing intermediate tables. + weight_column: The weight column name. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + worker_pool_specs_override: The dictionary for overriding training and + evaluation worker pool specs. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + run_evaluation: Whether to run evaluation steps during training. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + if isinstance(tf_auto_transform_features, list): + tf_auto_transform_features = {'auto': tf_auto_transform_features} + + if transform_config and tf_transformations_path: + raise ValueError( + 'Only one of transform_config and tf_transformations_path can ' + 'be specified.' + ) + + elif transform_config: + warnings.warn( + 'transform_config parameter is deprecated. ' + 'Please use the flattened transform config arguments instead.' + ) + tf_transformations_path = transform_config + + if not worker_pool_specs_override: + worker_pool_specs_override = [] + + parameter_values = {} + training_and_eval_parameters = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'prediction_type': prediction_type, + 'learning_rate': learning_rate, + 'dnn_learning_rate': dnn_learning_rate, + 'optimizer_type': optimizer_type, + 'max_steps': max_steps, + 'max_train_secs': max_train_secs, + 'l1_regularization_strength': l1_regularization_strength, + 'l2_regularization_strength': l2_regularization_strength, + 'l2_shrinkage_regularization_strength': ( + l2_shrinkage_regularization_strength + ), + 'beta_1': beta_1, + 'beta_2': beta_2, + 'hidden_units': hidden_units, + 'use_wide': use_wide, + 'embed_categories': embed_categories, + 'dnn_dropout': dnn_dropout, + 'dnn_optimizer_type': dnn_optimizer_type, + 'dnn_l1_regularization_strength': dnn_l1_regularization_strength, + 'dnn_l2_regularization_strength': dnn_l2_regularization_strength, + 'dnn_l2_shrinkage_regularization_strength': ( + dnn_l2_shrinkage_regularization_strength + ), + 'dnn_beta_1': dnn_beta_1, + 'dnn_beta_2': dnn_beta_2, + 'enable_profiler': enable_profiler, + 'cache_data': cache_data, + 'seed': seed, + 'eval_steps': eval_steps, + 'batch_size': batch_size, + 'measurement_selection_type': measurement_selection_type, + 'optimization_metric': optimization_metric, + 'eval_frequency_secs': eval_frequency_secs, + 'weight_column': weight_column, + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': transform_dataflow_max_num_workers, + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'worker_pool_specs_override': worker_pool_specs_override, + 'run_evaluation': run_evaluation, + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'dataflow_service_account': dataflow_service_account, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + } + _update_parameters(parameter_values, training_and_eval_parameters) + + fte_params = { + 'dataset_level_custom_transformation_definitions': ( + dataset_level_custom_transformation_definitions + if dataset_level_custom_transformation_definitions + else [] + ), + 'dataset_level_transformations': ( + dataset_level_transformations if dataset_level_transformations else [] + ), + 'run_feature_selection': run_feature_selection, + 'feature_selection_algorithm': feature_selection_algorithm, + 'max_selected_features': max_selected_features, + 'predefined_split_key': predefined_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'tf_auto_transform_features': ( + tf_auto_transform_features if tf_auto_transform_features else {} + ), + 'tf_custom_transformation_definitions': ( + tf_custom_transformation_definitions + if tf_custom_transformation_definitions + else [] + ), + 'tf_transformations_path': tf_transformations_path, + 'materialized_examples_format': ( + materialized_examples_format + if materialized_examples_format + else 'tfrecords_gzip' + ), + 'tf_transform_execution_engine': ( + tf_transform_execution_engine + if tf_transform_execution_engine + else 'dataflow' + ), + } + _update_parameters(parameter_values, fte_params) + + data_source_and_split_parameters = { + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_staging_full_dataset_id': bigquery_staging_full_dataset_id, + } + _update_parameters(parameter_values, data_source_and_split_parameters) + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'wide_and_deep_trainer_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_builtin_algorithm_hyperparameter_tuning_job_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + study_spec_metric_id: str, + study_spec_metric_goal: str, + study_spec_parameters_override: List[Dict[str, Any]], + max_trial_count: int, + parallel_trial_count: int, + algorithm: str, + enable_profiler: bool = False, + seed: int = 1, + eval_steps: int = 0, + eval_frequency_secs: int = 600, + transform_config: Optional[str] = None, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_transform_execution_engine: Optional[str] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + tf_custom_transformation_definitions: Optional[List[Dict[str, Any]]] = None, + tf_transformations_path: Optional[str] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + weight_column: str = '', + max_failed_trial_count: int = 0, + study_spec_algorithm: str = 'ALGORITHM_UNSPECIFIED', + study_spec_measurement_selection_type: str = 'BEST_MEASUREMENT', + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + worker_pool_specs_override: Optional[Dict[str, Any]] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: str = _EVALUATION_BATCH_PREDICT_MACHINE_TYPE, + evaluation_batch_predict_starting_replica_count: int = _EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT, + evaluation_batch_predict_max_replica_count: int = _EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT, + evaluation_dataflow_machine_type: str = _EVALUATION_DATAFLOW_MACHINE_TYPE, + evaluation_dataflow_starting_num_workers: int = _EVALUATION_DATAFLOW_STARTING_NUM_WORKERS, + evaluation_dataflow_max_num_workers: int = _EVALUATION_DATAFLOW_MAX_NUM_WORKERS, + evaluation_dataflow_disk_size_gb: int = _EVALUATION_DATAFLOW_DISK_SIZE_GB, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +) -> Tuple[str, Dict[str, Any]]: + """Get the built-in algorithm HyperparameterTuningJob pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + study_spec_metric_id: Metric to optimize, possible values: [ 'loss', + 'average_loss', 'rmse', 'mae', 'mql', 'accuracy', 'auc', 'precision', + 'recall']. + study_spec_metric_goal: Optimization goal of the metric, possible values: + "MAXIMIZE", "MINIMIZE". + study_spec_parameters_override: List of dictionaries representing parameters + to optimize. The dictionary key is the parameter_id, which is passed to + training job as a command line argument, and the dictionary value is the + parameter specification of the metric. + max_trial_count: The desired total number of trials. + parallel_trial_count: The desired number of trials to run in parallel. + algorithm: Algorithm to train. One of "tabnet" and "wide_and_deep". + enable_profiler: Enables profiling and saves a trace during evaluation. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or + negative, it means run evaluation on the whole validation dataset. If set + to 0, it means run evaluation for a fixed number of samples. + eval_frequency_secs: Frequency at which evaluation and checkpointing will + take place. + transform_config: Path to v1 TF transformation configuration. + dataset_level_custom_transformation_definitions: Dataset-level custom + transformation definitions in string format. + dataset_level_transformations: Dataset-level transformation configuration in + string format. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + training_fraction: Training fraction. + validation_fraction: Validation fraction. + test_fraction: Test fraction. + tf_transform_execution_engine: The execution engine used to execute TF-based + transformations. + tf_auto_transform_features: List of auto transform features in the + comma-separated string format. + tf_custom_transformation_definitions: TF custom transformation definitions + in string format. + tf_transformations_path: Path to TF transformation configuration. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + bigquery_staging_full_dataset_id: The BigQuery staging full dataset id for + storing intermediate tables. + weight_column: The weight column name. + max_failed_trial_count: The number of failed trials that need to be seen + before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides + how many trials must fail before the whole job fails. + study_spec_algorithm: The search algorithm specified for the study. One of + "ALGORITHM_UNSPECIFIED", "GRID_SEARCH", or "RANDOM_SEARCH". + study_spec_measurement_selection_type: Which measurement to use if/when the + service automatically selects the final measurement from previously + reported intermediate measurements. One of "BEST_MEASUREMENT" or + "LAST_MEASUREMENT". + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + worker_pool_specs_override: The dictionary for overriding training and + evaluation worker pool specs. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + run_evaluation: Whether to run evaluation steps during training. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + warnings.warn( + 'This method is deprecated. Please use' + ' get_tabnet_hyperparameter_tuning_job_pipeline_and_parameters or' + ' get_wide_and_deep_hyperparameter_tuning_job_pipeline_and_parameters' + ' instead.' + ) + + if algorithm == 'tabnet': + return get_tabnet_hyperparameter_tuning_job_pipeline_and_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + prediction_type=prediction_type, + study_spec_metric_id=study_spec_metric_id, + study_spec_metric_goal=study_spec_metric_goal, + study_spec_parameters_override=study_spec_parameters_override, + max_trial_count=max_trial_count, + parallel_trial_count=parallel_trial_count, + transform_config=transform_config, + dataset_level_custom_transformation_definitions=dataset_level_custom_transformation_definitions, + dataset_level_transformations=dataset_level_transformations, + predefined_split_key=predefined_split_key, + stratified_split_key=stratified_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + tf_transform_execution_engine=tf_transform_execution_engine, + tf_auto_transform_features=tf_auto_transform_features, + tf_custom_transformation_definitions=tf_custom_transformation_definitions, + tf_transformations_path=tf_transformations_path, + enable_profiler=enable_profiler, + seed=seed, + eval_steps=eval_steps, + eval_frequency_secs=eval_frequency_secs, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + bigquery_staging_full_dataset_id=bigquery_staging_full_dataset_id, + weight_column=weight_column, + max_failed_trial_count=max_failed_trial_count, + study_spec_algorithm=study_spec_algorithm, + study_spec_measurement_selection_type=study_spec_measurement_selection_type, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + worker_pool_specs_override=worker_pool_specs_override, + run_evaluation=run_evaluation, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + ) + elif algorithm == 'wide_and_deep': + return get_wide_and_deep_hyperparameter_tuning_job_pipeline_and_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + prediction_type=prediction_type, + study_spec_metric_id=study_spec_metric_id, + study_spec_metric_goal=study_spec_metric_goal, + study_spec_parameters_override=study_spec_parameters_override, + max_trial_count=max_trial_count, + parallel_trial_count=parallel_trial_count, + transform_config=transform_config, + dataset_level_custom_transformation_definitions=dataset_level_custom_transformation_definitions, + dataset_level_transformations=dataset_level_transformations, + predefined_split_key=predefined_split_key, + stratified_split_key=stratified_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + tf_transform_execution_engine=tf_transform_execution_engine, + tf_auto_transform_features=tf_auto_transform_features, + tf_custom_transformation_definitions=tf_custom_transformation_definitions, + tf_transformations_path=tf_transformations_path, + enable_profiler=enable_profiler, + seed=seed, + eval_steps=eval_steps, + eval_frequency_secs=eval_frequency_secs, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + bigquery_staging_full_dataset_id=bigquery_staging_full_dataset_id, + weight_column=weight_column, + max_failed_trial_count=max_failed_trial_count, + study_spec_algorithm=study_spec_algorithm, + study_spec_measurement_selection_type=study_spec_measurement_selection_type, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + worker_pool_specs_override=worker_pool_specs_override, + run_evaluation=run_evaluation, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + ) + else: + raise ValueError( + 'Invalid algorithm provided. Supported values are "tabnet" and' + ' "wide_and_deep".' + ) + + +def get_tabnet_hyperparameter_tuning_job_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + study_spec_metric_id: str, + study_spec_metric_goal: str, + study_spec_parameters_override: List[Dict[str, Any]], + max_trial_count: int, + parallel_trial_count: int, + transform_config: Optional[str] = None, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + run_feature_selection: bool = False, + feature_selection_algorithm: Optional[str] = None, + materialized_examples_format: Optional[str] = None, + max_selected_features: Optional[int] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_transform_execution_engine: Optional[str] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + tf_custom_transformation_definitions: Optional[List[Dict[str, Any]]] = None, + tf_transformations_path: Optional[str] = None, + enable_profiler: bool = False, + cache_data: str = 'auto', + seed: int = 1, + eval_steps: int = 0, + eval_frequency_secs: int = 600, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + weight_column: str = '', + max_failed_trial_count: int = 0, + study_spec_algorithm: str = 'ALGORITHM_UNSPECIFIED', + study_spec_measurement_selection_type: str = 'BEST_MEASUREMENT', + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + worker_pool_specs_override: Optional[Dict[str, Any]] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: str = _EVALUATION_BATCH_PREDICT_MACHINE_TYPE, + evaluation_batch_predict_starting_replica_count: int = _EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT, + evaluation_batch_predict_max_replica_count: int = _EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT, + evaluation_dataflow_machine_type: str = _EVALUATION_DATAFLOW_MACHINE_TYPE, + evaluation_dataflow_starting_num_workers: int = _EVALUATION_DATAFLOW_STARTING_NUM_WORKERS, + evaluation_dataflow_max_num_workers: int = _EVALUATION_DATAFLOW_MAX_NUM_WORKERS, + evaluation_dataflow_disk_size_gb: int = _EVALUATION_DATAFLOW_DISK_SIZE_GB, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +) -> Tuple[str, Dict[str, Any]]: + """Get the TabNet HyperparameterTuningJob pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + study_spec_metric_id: Metric to optimize, possible values: [ 'loss', + 'average_loss', 'rmse', 'mae', 'mql', 'accuracy', 'auc', 'precision', + 'recall']. + study_spec_metric_goal: Optimization goal of the metric, possible values: + "MAXIMIZE", "MINIMIZE". + study_spec_parameters_override: List of dictionaries representing parameters + to optimize. The dictionary key is the parameter_id, which is passed to + training job as a command line argument, and the dictionary value is the + parameter specification of the metric. + max_trial_count: The desired total number of trials. + parallel_trial_count: The desired number of trials to run in parallel. + transform_config: Path to v1 TF transformation configuration. + dataset_level_custom_transformation_definitions: Dataset-level custom + transformation definitions in string format. + dataset_level_transformations: Dataset-level transformation configuration in + string format. + run_feature_selection: Whether to enable feature selection. + feature_selection_algorithm: Feature selection algorithm. + materialized_examples_format: The format for the materialized examples. + max_selected_features: Maximum number of features to select. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + training_fraction: Training fraction. + validation_fraction: Validation fraction. + test_fraction: Test fraction. + tf_transform_execution_engine: The execution engine used to execute TF-based + transformations. + tf_auto_transform_features: List of auto transform features in the + comma-separated string format. + tf_custom_transformation_definitions: TF custom transformation definitions + in string format. + tf_transformations_path: Path to TF transformation configuration. + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is + determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or + negative, it means run evaluation on the whole validation dataset. If set + to 0, it means run evaluation for a fixed number of samples. + eval_frequency_secs: Frequency at which evaluation and checkpointing will + take place. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + bigquery_staging_full_dataset_id: The BigQuery staging full dataset id for + storing intermediate tables. + weight_column: The weight column name. + max_failed_trial_count: The number of failed trials that need to be seen + before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides + how many trials must fail before the whole job fails. + study_spec_algorithm: The search algorithm specified for the study. One of + "ALGORITHM_UNSPECIFIED", "GRID_SEARCH", or "RANDOM_SEARCH". + study_spec_measurement_selection_type: Which measurement to use if/when the + service automatically selects the final measurement from previously + reported intermediate measurements. One of "BEST_MEASUREMENT" or + "LAST_MEASUREMENT". + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + worker_pool_specs_override: The dictionary for overriding training and + evaluation worker pool specs. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + run_evaluation: Whether to run evaluation steps during training. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + if isinstance(tf_auto_transform_features, list): + tf_auto_transform_features = {'auto': tf_auto_transform_features} + + if transform_config and tf_transformations_path: + raise ValueError( + 'Only one of transform_config and tf_transformations_path can ' + 'be specified.' + ) + + elif transform_config: + warnings.warn( + 'transform_config parameter is deprecated. ' + 'Please use the flattened transform config arguments instead.' + ) + tf_transformations_path = transform_config + + if not worker_pool_specs_override: + worker_pool_specs_override = [] + + parameter_values = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'prediction_type': prediction_type, + 'study_spec_metric_id': study_spec_metric_id, + 'study_spec_metric_goal': study_spec_metric_goal, + 'study_spec_parameters_override': study_spec_parameters_override, + 'max_trial_count': max_trial_count, + 'parallel_trial_count': parallel_trial_count, + 'enable_profiler': enable_profiler, + 'cache_data': cache_data, + 'seed': seed, + 'eval_steps': eval_steps, + 'eval_frequency_secs': eval_frequency_secs, + 'weight_column': weight_column, + 'max_failed_trial_count': max_failed_trial_count, + 'study_spec_algorithm': study_spec_algorithm, + 'study_spec_measurement_selection_type': ( + study_spec_measurement_selection_type + ), + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': transform_dataflow_max_num_workers, + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'worker_pool_specs_override': worker_pool_specs_override, + 'run_evaluation': run_evaluation, + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'dataflow_service_account': dataflow_service_account, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + } + + fte_params = { + 'dataset_level_custom_transformation_definitions': ( + dataset_level_custom_transformation_definitions + if dataset_level_custom_transformation_definitions + else [] + ), + 'dataset_level_transformations': ( + dataset_level_transformations if dataset_level_transformations else [] + ), + 'run_feature_selection': run_feature_selection, + 'feature_selection_algorithm': feature_selection_algorithm, + 'max_selected_features': max_selected_features, + 'predefined_split_key': predefined_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'tf_auto_transform_features': ( + tf_auto_transform_features if tf_auto_transform_features else {} + ), + 'tf_custom_transformation_definitions': ( + tf_custom_transformation_definitions + if tf_custom_transformation_definitions + else [] + ), + 'tf_transformations_path': tf_transformations_path, + 'materialized_examples_format': ( + materialized_examples_format + if materialized_examples_format + else 'tfrecords_gzip' + ), + 'tf_transform_execution_engine': ( + tf_transform_execution_engine + if tf_transform_execution_engine + else 'dataflow' + ), + } + _update_parameters(parameter_values, fte_params) + + data_source_and_split_parameters = { + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_staging_full_dataset_id': bigquery_staging_full_dataset_id, + } + _update_parameters(parameter_values, data_source_and_split_parameters) + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'tabnet_hyperparameter_tuning_job_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_wide_and_deep_hyperparameter_tuning_job_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + study_spec_metric_id: str, + study_spec_metric_goal: str, + study_spec_parameters_override: List[Dict[str, Any]], + max_trial_count: int, + parallel_trial_count: int, + transform_config: Optional[str] = None, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + run_feature_selection: bool = False, + feature_selection_algorithm: Optional[str] = None, + materialized_examples_format: Optional[str] = None, + max_selected_features: Optional[int] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_transform_execution_engine: Optional[str] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + tf_custom_transformation_definitions: Optional[List[Dict[str, Any]]] = None, + tf_transformations_path: Optional[str] = None, + enable_profiler: bool = False, + cache_data: str = 'auto', + seed: int = 1, + eval_steps: int = 0, + eval_frequency_secs: int = 600, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + weight_column: str = '', + max_failed_trial_count: int = 0, + study_spec_algorithm: str = 'ALGORITHM_UNSPECIFIED', + study_spec_measurement_selection_type: str = 'BEST_MEASUREMENT', + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + worker_pool_specs_override: Optional[Dict[str, Any]] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: str = _EVALUATION_BATCH_PREDICT_MACHINE_TYPE, + evaluation_batch_predict_starting_replica_count: int = _EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT, + evaluation_batch_predict_max_replica_count: int = _EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT, + evaluation_dataflow_machine_type: str = _EVALUATION_DATAFLOW_MACHINE_TYPE, + evaluation_dataflow_starting_num_workers: int = _EVALUATION_DATAFLOW_STARTING_NUM_WORKERS, + evaluation_dataflow_max_num_workers: int = _EVALUATION_DATAFLOW_MAX_NUM_WORKERS, + evaluation_dataflow_disk_size_gb: int = _EVALUATION_DATAFLOW_DISK_SIZE_GB, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +) -> Tuple[str, Dict[str, Any]]: + """Get the Wide & Deep algorithm HyperparameterTuningJob pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + study_spec_metric_id: Metric to optimize, possible values: [ 'loss', + 'average_loss', 'rmse', 'mae', 'mql', 'accuracy', 'auc', 'precision', + 'recall']. + study_spec_metric_goal: Optimization goal of the metric, possible values: + "MAXIMIZE", "MINIMIZE". + study_spec_parameters_override: List of dictionaries representing parameters + to optimize. The dictionary key is the parameter_id, which is passed to + training job as a command line argument, and the dictionary value is the + parameter specification of the metric. + max_trial_count: The desired total number of trials. + parallel_trial_count: The desired number of trials to run in parallel. + transform_config: Path to v1 TF transformation configuration. + dataset_level_custom_transformation_definitions: Dataset-level custom + transformation definitions in string format. + dataset_level_transformations: Dataset-level transformation configuration in + string format. + run_feature_selection: Whether to enable feature selection. + feature_selection_algorithm: Feature selection algorithm. + materialized_examples_format: The format for the materialized examples. + max_selected_features: Maximum number of features to select. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + training_fraction: Training fraction. + validation_fraction: Validation fraction. + test_fraction: Test fraction. + tf_transform_execution_engine: The execution engine used to execute TF-based + transformations. + tf_auto_transform_features: List of auto transform features in the + comma-separated string format. + tf_custom_transformation_definitions: TF custom transformation definitions + in string format. + tf_transformations_path: Path to TF transformation configuration. + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is + determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or + negative, it means run evaluation on the whole validation dataset. If set + to 0, it means run evaluation for a fixed number of samples. + eval_frequency_secs: Frequency at which evaluation and checkpointing will + take place. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + bigquery_staging_full_dataset_id: The BigQuery staging full dataset id for + storing intermediate tables. + weight_column: The weight column name. + max_failed_trial_count: The number of failed trials that need to be seen + before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides + how many trials must fail before the whole job fails. + study_spec_algorithm: The search algorithm specified for the study. One of + "ALGORITHM_UNSPECIFIED", "GRID_SEARCH", or "RANDOM_SEARCH". + study_spec_measurement_selection_type: Which measurement to use if/when the + service automatically selects the final measurement from previously + reported intermediate measurements. One of "BEST_MEASUREMENT" or + "LAST_MEASUREMENT". + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + worker_pool_specs_override: The dictionary for overriding training and + evaluation worker pool specs. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + run_evaluation: Whether to run evaluation steps during training. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + if isinstance(tf_auto_transform_features, list): + tf_auto_transform_features = {'auto': tf_auto_transform_features} + + if transform_config and tf_transformations_path: + raise ValueError( + 'Only one of transform_config and tf_transformations_path can ' + 'be specified.' + ) + + elif transform_config: + warnings.warn( + 'transform_config parameter is deprecated. ' + 'Please use the flattened transform config arguments instead.' + ) + tf_transformations_path = transform_config + + if not worker_pool_specs_override: + worker_pool_specs_override = [] + + parameter_values = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'prediction_type': prediction_type, + 'study_spec_metric_id': study_spec_metric_id, + 'study_spec_metric_goal': study_spec_metric_goal, + 'study_spec_parameters_override': study_spec_parameters_override, + 'max_trial_count': max_trial_count, + 'parallel_trial_count': parallel_trial_count, + 'enable_profiler': enable_profiler, + 'cache_data': cache_data, + 'seed': seed, + 'eval_steps': eval_steps, + 'eval_frequency_secs': eval_frequency_secs, + 'weight_column': weight_column, + 'max_failed_trial_count': max_failed_trial_count, + 'study_spec_algorithm': study_spec_algorithm, + 'study_spec_measurement_selection_type': ( + study_spec_measurement_selection_type + ), + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': transform_dataflow_max_num_workers, + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'worker_pool_specs_override': worker_pool_specs_override, + 'run_evaluation': run_evaluation, + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'dataflow_service_account': dataflow_service_account, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + } + + fte_params = { + 'dataset_level_custom_transformation_definitions': ( + dataset_level_custom_transformation_definitions + if dataset_level_custom_transformation_definitions + else [] + ), + 'dataset_level_transformations': ( + dataset_level_transformations if dataset_level_transformations else [] + ), + 'run_feature_selection': run_feature_selection, + 'feature_selection_algorithm': feature_selection_algorithm, + 'max_selected_features': max_selected_features, + 'predefined_split_key': predefined_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'tf_auto_transform_features': ( + tf_auto_transform_features if tf_auto_transform_features else {} + ), + 'tf_custom_transformation_definitions': ( + tf_custom_transformation_definitions + if tf_custom_transformation_definitions + else [] + ), + 'tf_transformations_path': tf_transformations_path, + 'materialized_examples_format': ( + materialized_examples_format + if materialized_examples_format + else 'tfrecords_gzip' + ), + 'tf_transform_execution_engine': ( + tf_transform_execution_engine + if tf_transform_execution_engine + else 'dataflow' + ), + } + _update_parameters(parameter_values, fte_params) + + data_source_and_split_parameters = { + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_staging_full_dataset_id': bigquery_staging_full_dataset_id, + } + _update_parameters(parameter_values, data_source_and_split_parameters) + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'wide_and_deep_hyperparameter_tuning_job_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_tabnet_trainer_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + learning_rate: float, + transform_config: Optional[str] = None, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + run_feature_selection: bool = False, + feature_selection_algorithm: Optional[str] = None, + materialized_examples_format: Optional[str] = None, + max_selected_features: Optional[int] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_transform_execution_engine: Optional[str] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + tf_custom_transformation_definitions: Optional[List[Dict[str, Any]]] = None, + tf_transformations_path: Optional[str] = None, + max_steps: int = -1, + max_train_secs: int = -1, + large_category_dim: int = 1, + large_category_thresh: int = 300, + yeo_johnson_transform: bool = True, + feature_dim: int = 64, + feature_dim_ratio: float = 0.5, + num_decision_steps: int = 6, + relaxation_factor: float = 1.5, + decay_every: float = 100, + decay_rate: float = 0.95, + gradient_thresh: float = 2000, + sparsity_loss_weight: float = 0.00001, + batch_momentum: float = 0.95, + batch_size_ratio: float = 0.25, + num_transformer_layers: int = 4, + num_transformer_layers_ratio: float = 0.25, + class_weight: float = 1.0, + loss_function_type: str = 'default', + alpha_focal_loss: float = 0.25, + gamma_focal_loss: float = 2.0, + enable_profiler: bool = False, + cache_data: str = 'auto', + seed: int = 1, + eval_steps: int = 0, + batch_size: int = 100, + measurement_selection_type: Optional[str] = None, + optimization_metric: Optional[str] = None, + eval_frequency_secs: int = 600, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + weight_column: str = '', + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + worker_pool_specs_override: Optional[Dict[str, Any]] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: str = _EVALUATION_BATCH_PREDICT_MACHINE_TYPE, + evaluation_batch_predict_starting_replica_count: int = _EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT, + evaluation_batch_predict_max_replica_count: int = _EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT, + evaluation_dataflow_machine_type: str = _EVALUATION_DATAFLOW_MACHINE_TYPE, + evaluation_dataflow_starting_num_workers: int = _EVALUATION_DATAFLOW_STARTING_NUM_WORKERS, + evaluation_dataflow_max_num_workers: int = _EVALUATION_DATAFLOW_MAX_NUM_WORKERS, + evaluation_dataflow_disk_size_gb: int = _EVALUATION_DATAFLOW_DISK_SIZE_GB, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', +) -> Tuple[str, Dict[str, Any]]: + """Get the TabNet training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + learning_rate: The learning rate used by the linear optimizer. + transform_config: Path to v1 TF transformation configuration. + dataset_level_custom_transformation_definitions: Dataset-level custom + transformation definitions in string format. + dataset_level_transformations: Dataset-level transformation configuration in + string format. + run_feature_selection: Whether to enable feature selection. + feature_selection_algorithm: Feature selection algorithm. + materialized_examples_format: The format for the materialized examples. + max_selected_features: Maximum number of features to select. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + training_fraction: Training fraction. + validation_fraction: Validation fraction. + test_fraction: Test fraction. + tf_transform_execution_engine: The execution engine used to execute TF-based + transformations. + tf_auto_transform_features: List of auto transform features in the + comma-separated string format. + tf_custom_transformation_definitions: TF custom transformation definitions + in string format. + tf_transformations_path: Path to TF transformation configuration. + max_steps: Number of steps to run the trainer for. + max_train_secs: Amount of time in seconds to run the trainer for. + large_category_dim: Embedding dimension for categorical feature with large + number of categories. + large_category_thresh: Threshold for number of categories to apply + large_category_dim embedding dimension to. + yeo_johnson_transform: Enables trainable Yeo-Johnson power transform. + feature_dim: Dimensionality of the hidden representation in feature + transformation block. + feature_dim_ratio: The ratio of output dimension (dimensionality of the + outputs of each decision step) to feature dimension. + num_decision_steps: Number of sequential decision steps. + relaxation_factor: Relaxation factor that promotes the reuse of each feature + at different decision steps. When it is 1, a feature is enforced to be + used only at one decision step and as it increases, more flexibility is + provided to use a feature at multiple decision steps. + decay_every: Number of iterations for periodically applying learning rate + decaying. + decay_rate: Learning rate decaying. + gradient_thresh: Threshold for the norm of gradients for clipping. + sparsity_loss_weight: Weight of the loss for sparsity regularization + (increasing it will yield more sparse feature selection). + batch_momentum: Momentum in ghost batch normalization. + batch_size_ratio: The ratio of virtual batch size (size of the ghost batch + normalization) to batch size. + num_transformer_layers: The number of transformer layers for each decision + step. used only at one decision step and as it increases, more flexibility + is provided to use a feature at multiple decision steps. + num_transformer_layers_ratio: The ratio of shared transformer layer to + transformer layers. + class_weight: The class weight is used to computes a weighted cross entropy + which is helpful in classify imbalanced dataset. Only used for + classification. + loss_function_type: Loss function type. Loss function in classification + [cross_entropy, weighted_cross_entropy, focal_loss], default is + cross_entropy. Loss function in regression: [rmse, mae, mse], default is + mse. + alpha_focal_loss: Alpha value (balancing factor) in focal_loss function. + Only used for classification. + gamma_focal_loss: Gamma value (modulating factor) for focal loss for focal + loss. Only used for classification. + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is + determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or + negative, it means run evaluation on the whole validation dataset. If set + to 0, it means run evaluation for a fixed number of samples. + batch_size: Batch size for training. + measurement_selection_type: Which measurement to use if/when the service + automatically selects the final measurement from previously reported + intermediate measurements. One of "BEST_MEASUREMENT" or + "LAST_MEASUREMENT". + optimization_metric: Optimization metric used for + `measurement_selection_type`. Default is "rmse" for regression and "auc" + for classification. + eval_frequency_secs: Frequency at which evaluation and checkpointing will + take place. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + bigquery_staging_full_dataset_id: The BigQuery staging full dataset id for + storing intermediate tables. + weight_column: The weight column name. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + worker_pool_specs_override: The dictionary for overriding training and + evaluation worker pool specs. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + run_evaluation: Whether to run evaluation steps during training. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + if isinstance(tf_auto_transform_features, list): + tf_auto_transform_features = {'auto': tf_auto_transform_features} + + if transform_config and tf_transformations_path: + raise ValueError( + 'Only one of transform_config and tf_transformations_path can ' + 'be specified.' + ) + + elif transform_config: + warnings.warn( + 'transform_config parameter is deprecated. ' + 'Please use the flattened transform config arguments instead.' + ) + tf_transformations_path = transform_config + + if not worker_pool_specs_override: + worker_pool_specs_override = [] + + parameter_values = {} + training_and_eval_parameters = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'prediction_type': prediction_type, + 'learning_rate': learning_rate, + 'max_steps': max_steps, + 'max_train_secs': max_train_secs, + 'large_category_dim': large_category_dim, + 'large_category_thresh': large_category_thresh, + 'yeo_johnson_transform': yeo_johnson_transform, + 'feature_dim': feature_dim, + 'feature_dim_ratio': feature_dim_ratio, + 'num_decision_steps': num_decision_steps, + 'relaxation_factor': relaxation_factor, + 'decay_every': decay_every, + 'decay_rate': decay_rate, + 'gradient_thresh': gradient_thresh, + 'sparsity_loss_weight': sparsity_loss_weight, + 'batch_momentum': batch_momentum, + 'batch_size_ratio': batch_size_ratio, + 'num_transformer_layers': num_transformer_layers, + 'num_transformer_layers_ratio': num_transformer_layers_ratio, + 'class_weight': class_weight, + 'loss_function_type': loss_function_type, + 'alpha_focal_loss': alpha_focal_loss, + 'gamma_focal_loss': gamma_focal_loss, + 'enable_profiler': enable_profiler, + 'cache_data': cache_data, + 'seed': seed, + 'eval_steps': eval_steps, + 'batch_size': batch_size, + 'measurement_selection_type': measurement_selection_type, + 'optimization_metric': optimization_metric, + 'eval_frequency_secs': eval_frequency_secs, + 'weight_column': weight_column, + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': transform_dataflow_max_num_workers, + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'worker_pool_specs_override': worker_pool_specs_override, + 'run_evaluation': run_evaluation, + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'dataflow_service_account': dataflow_service_account, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + } + _update_parameters(parameter_values, training_and_eval_parameters) + + fte_params = { + 'dataset_level_custom_transformation_definitions': ( + dataset_level_custom_transformation_definitions + if dataset_level_custom_transformation_definitions + else [] + ), + 'dataset_level_transformations': ( + dataset_level_transformations if dataset_level_transformations else [] + ), + 'run_feature_selection': run_feature_selection, + 'feature_selection_algorithm': feature_selection_algorithm, + 'max_selected_features': max_selected_features, + 'predefined_split_key': predefined_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'tf_auto_transform_features': ( + tf_auto_transform_features if tf_auto_transform_features else {} + ), + 'tf_custom_transformation_definitions': ( + tf_custom_transformation_definitions + if tf_custom_transformation_definitions + else [] + ), + 'tf_transformations_path': tf_transformations_path, + 'materialized_examples_format': ( + materialized_examples_format + if materialized_examples_format + else 'tfrecords_gzip' + ), + 'tf_transform_execution_engine': ( + tf_transform_execution_engine + if tf_transform_execution_engine + else 'dataflow' + ), + } + _update_parameters(parameter_values, fte_params) + + data_source_and_split_parameters = { + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_staging_full_dataset_id': bigquery_staging_full_dataset_id, + } + _update_parameters(parameter_values, data_source_and_split_parameters) + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), 'tabnet_trainer_pipeline.yaml' + ) + + return pipeline_definition_path, parameter_values + + +def get_tabnet_study_spec_parameters_override( + dataset_size_bucket: str, prediction_type: str, training_budget_bucket: str +) -> List[Dict[str, Any]]: + """Get study_spec_parameters_override for a TabNet hyperparameter tuning job. + + Args: + dataset_size_bucket: Size of the dataset. One of "small" (< 1M rows), + "medium" (1M - 100M rows), or "large" (> 100M rows). + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + training_budget_bucket: Bucket of the estimated training budget. One of + "small" (< $600), "medium" ($600 - $2400), or "large" (> $2400). This + parameter is only used as a hint for the hyperparameter search space, + unrelated to the real cost. + + Returns: + List of study_spec_parameters_override. + """ + + if dataset_size_bucket not in ['small', 'medium', 'large']: + raise ValueError( + 'Invalid dataset_size_bucket provided. Supported values ' + ' are "small", "medium" or "large".' + ) + if training_budget_bucket not in ['small', 'medium', 'large']: + raise ValueError( + 'Invalid training_budget_bucket provided. Supported values ' + 'are "small", "medium" or "large".' + ) + + param_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + f'configs/tabnet_params_{dataset_size_bucket}_data_{training_budget_bucket}_search_space.json', + ) + with open(param_path, 'r') as f: + param_content = f.read() + params = json.loads(param_content) + + if prediction_type == 'regression': + return _format_tabnet_regression_study_spec_parameters_override( + params, training_budget_bucket + ) + return params + + +def _format_tabnet_regression_study_spec_parameters_override( + params: List[Dict[str, Any]], training_budget_bucket: str +) -> List[Dict[str, Any]]: + """Get regression study_spec_parameters_override for a TabNet hyperparameter tuning job. + + Args: + params: List of dictionaries representing parameters to optimize. The + dictionary key is the parameter_id, which is passed to training job as a + command line argument, and the dictionary value is the parameter + specification of the metric. + training_budget_bucket: Bucket of the estimated training budget. One of + "small" (< $600), "medium" ($600 - $2400), or "large" (> $2400). This + parameter is only used as a hint for the hyperparameter search space, + unrelated to the real cost. + + Returns: + List of study_spec_parameters_override for regression. + """ + + # To get regression study_spec_parameters, we need to set + # `loss_function_type` to ‘mae’ (‘mae’ and ‘mse’ for "large" search space), + # remove the `alpha_focal_loss`, `gamma_focal_loss` + # and `class_weight` parameters and increase the max for + # `sparsity_loss_weight` to 100. + formatted_params = [] + for param in params: + if param['parameter_id'] in [ + 'alpha_focal_loss', + 'gamma_focal_loss', + 'class_weight', + ]: + continue + elif param['parameter_id'] == 'sparsity_loss_weight': + param['double_value_spec']['max_value'] = 100 + elif param['parameter_id'] == 'loss_function_type': + if training_budget_bucket == 'large': + param['categorical_value_spec']['values'] = ['mae', 'mse'] + else: + param['categorical_value_spec']['values'] = ['mae'] + + formatted_params.append(param) + + return formatted_params + + +def get_wide_and_deep_study_spec_parameters_override() -> List[Dict[str, Any]]: + """Get study_spec_parameters_override for a Wide & Deep hyperparameter tuning job. + + Returns: + List of study_spec_parameters_override. + """ + param_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'configs/wide_and_deep_params.json', + ) + with open(param_path, 'r') as f: + param_content = f.read() + params = json.loads(param_content) + + return params + + +def get_xgboost_study_spec_parameters_override() -> List[Dict[str, Any]]: + """Get study_spec_parameters_override for an XGBoost hyperparameter tuning job. + + Returns: + List of study_spec_parameters_override. + """ + param_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), 'configs/xgboost_params.json' + ) + with open(param_path, 'r') as f: + param_content = f.read() + params = json.loads(param_content) + + return params + + +def get_xgboost_trainer_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + objective: str, + eval_metric: Optional[str] = None, + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + base_score: Optional[float] = None, + disable_default_eval_metric: Optional[int] = None, + seed: Optional[int] = None, + seed_per_iteration: Optional[bool] = None, + booster: Optional[str] = None, + eta: Optional[float] = None, + gamma: Optional[float] = None, + max_depth: Optional[int] = None, + min_child_weight: Optional[float] = None, + max_delta_step: Optional[float] = None, + subsample: Optional[float] = None, + colsample_bytree: Optional[float] = None, + colsample_bylevel: Optional[float] = None, + colsample_bynode: Optional[float] = None, + reg_lambda: Optional[float] = None, + reg_alpha: Optional[float] = None, + tree_method: Optional[str] = None, + scale_pos_weight: Optional[float] = None, + updater: Optional[str] = None, + refresh_leaf: Optional[int] = None, + process_type: Optional[str] = None, + grow_policy: Optional[str] = None, + sampling_method: Optional[str] = None, + monotone_constraints: Optional[str] = None, + interaction_constraints: Optional[str] = None, + sample_type: Optional[str] = None, + normalize_type: Optional[str] = None, + rate_drop: Optional[float] = None, + one_drop: Optional[int] = None, + skip_drop: Optional[float] = None, + num_parallel_tree: Optional[int] = None, + feature_selector: Optional[str] = None, + top_k: Optional[int] = None, + max_cat_to_onehot: Optional[int] = None, + max_leaves: Optional[int] = None, + max_bin: Optional[int] = None, + tweedie_variance_power: Optional[float] = None, + huber_slope: Optional[float] = None, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + run_feature_selection: Optional[bool] = None, + feature_selection_algorithm: Optional[str] = None, + max_selected_features: Optional[int] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + tf_custom_transformation_definitions: Optional[List[Dict[str, Any]]] = None, + tf_transformations_path: Optional[str] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + weight_column: Optional[str] = None, + training_machine_type: Optional[str] = None, + training_total_replica_count: Optional[int] = None, + training_accelerator_type: Optional[str] = None, + training_accelerator_count: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + run_evaluation: Optional[bool] = None, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, + dataflow_service_account: Optional[str] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: Optional[bool] = None, + encryption_spec_key_name: Optional[str] = None, +): + """Get the XGBoost training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + objective: Specifies the learning task and the learning objective. Must be + one of [reg:squarederror, reg:squaredlogerror, + reg:logistic, reg:gamma, reg:tweedie, reg:pseudohubererror, + binary:logistic, multi:softprob]. + eval_metric: Evaluation metrics for validation data represented as a + comma-separated string. + num_boost_round: Number of boosting iterations. + early_stopping_rounds: Activates early stopping. Validation error needs to + decrease at least every early_stopping_rounds round(s) to continue + training. + base_score: The initial prediction score of all instances, global bias. + disable_default_eval_metric: Flag to disable default metric. Set to >0 to + disable. Default to 0. + seed: Random seed. + seed_per_iteration: Seed PRNG determnisticly via iterator number. + booster: Which booster to use, can be gbtree, gblinear or dart. gbtree and + dart use tree based model while gblinear uses linear function. + eta: Learning rate. + gamma: Minimum loss reduction required to make a further partition on a leaf + node of the tree. + max_depth: Maximum depth of a tree. + min_child_weight: Minimum sum of instance weight(hessian) needed in a child. + max_delta_step: Maximum delta step we allow each tree's weight estimation to + be. + subsample: Subsample ratio of the training instance. + colsample_bytree: Subsample ratio of columns when constructing each tree. + colsample_bylevel: Subsample ratio of columns for each split, in each level. + colsample_bynode: Subsample ratio of columns for each node (split). + reg_lambda: L2 regularization term on weights. + reg_alpha: L1 regularization term on weights. + tree_method: The tree construction algorithm used in XGBoost. Choices: + ["auto", "exact", "approx", "hist", "gpu_exact", "gpu_hist"]. + scale_pos_weight: Control the balance of positive and negative weights. + updater: A comma separated string defining the sequence of tree updaters to + run. + refresh_leaf: Refresh updater plugin. Update tree leaf and nodes's stats if + True. When it is False, only node stats are updated. + process_type: A type of boosting process to run. Choices:["default", + "update"] + grow_policy: Controls a way new nodes are added to the tree. Only supported + if tree_method is hist. Choices:["depthwise", "lossguide"] + sampling_method: The method to use to sample the training instances. + monotone_constraints: Constraint of variable monotonicity. + interaction_constraints: Constraints for interaction representing permitted + interactions. + sample_type: [dart booster only] Type of sampling algorithm. + Choices:["uniform", "weighted"] + normalize_type: [dart booster only] Type of normalization algorithm, + Choices:["tree", "forest"] + rate_drop: [dart booster only] Dropout rate.' + one_drop: [dart booster only] When this flag is enabled, at least one tree + is always dropped during the dropout (allows Binomial-plus-one or + epsilon-dropout from the original DART paper). + skip_drop: [dart booster only] Probability of skipping the dropout procedure + during a boosting iteration. + num_parallel_tree: Number of parallel trees constructed during each + iteration. This option is used to support boosted random forest. + feature_selector: [linear booster only] Feature selection and ordering + method. + top_k: The number of top features to select in greedy and thrifty feature + selector. The value of 0 means using all the features. + max_cat_to_onehot: A threshold for deciding whether XGBoost should use + one-hot encoding based split for categorical data. + max_leaves: Maximum number of nodes to be added. + max_bin: Maximum number of discrete bins to bucket continuous features. + tweedie_variance_power: Parameter that controls the variance of the Tweedie + distribution. + huber_slope: A parameter used for Pseudo-Huber loss to define the delta + term. + dataset_level_custom_transformation_definitions: Dataset-level custom + transformation definitions in string format. + dataset_level_transformations: Dataset-level transformation configuration in + string format. + run_feature_selection: Whether to enable feature selection. + feature_selection_algorithm: Feature selection algorithm. + max_selected_features: Maximum number of features to select. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + training_fraction: Training fraction. + validation_fraction: Validation fraction. + test_fraction: Test fraction. + tf_auto_transform_features: List of auto transform features in the + comma-separated string format. + tf_custom_transformation_definitions: TF custom transformation definitions + in string format. + tf_transformations_path: Path to TF transformation configuration. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + bigquery_staging_full_dataset_id: The BigQuery staging full dataset id for + storing intermediate tables. + weight_column: The weight column name. + training_machine_type: Machine type. + training_total_replica_count: Number of workers. + training_accelerator_type: Accelerator type. + training_accelerator_count: Accelerator count. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + run_evaluation: Whether to run evaluation steps during training. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = {} + if isinstance(tf_auto_transform_features, list): + tf_auto_transform_features = {'auto': tf_auto_transform_features} + + training_and_eval_parameters = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'objective': objective, + 'eval_metric': eval_metric, + 'num_boost_round': num_boost_round, + 'early_stopping_rounds': early_stopping_rounds, + 'base_score': base_score, + 'disable_default_eval_metric': disable_default_eval_metric, + 'seed': seed, + 'seed_per_iteration': seed_per_iteration, + 'booster': booster, + 'eta': eta, + 'gamma': gamma, + 'max_depth': max_depth, + 'min_child_weight': min_child_weight, + 'max_delta_step': max_delta_step, + 'subsample': subsample, + 'colsample_bytree': colsample_bytree, + 'colsample_bylevel': colsample_bylevel, + 'colsample_bynode': colsample_bynode, + 'reg_lambda': reg_lambda, + 'reg_alpha': reg_alpha, + 'tree_method': tree_method, + 'scale_pos_weight': scale_pos_weight, + 'updater': updater, + 'refresh_leaf': refresh_leaf, + 'process_type': process_type, + 'grow_policy': grow_policy, + 'sampling_method': sampling_method, + 'monotone_constraints': monotone_constraints, + 'interaction_constraints': interaction_constraints, + 'sample_type': sample_type, + 'normalize_type': normalize_type, + 'rate_drop': rate_drop, + 'one_drop': one_drop, + 'skip_drop': skip_drop, + 'num_parallel_tree': num_parallel_tree, + 'feature_selector': feature_selector, + 'top_k': top_k, + 'max_cat_to_onehot': max_cat_to_onehot, + 'max_leaves': max_leaves, + 'max_bin': max_bin, + 'tweedie_variance_power': tweedie_variance_power, + 'huber_slope': huber_slope, + 'weight_column': weight_column, + 'training_machine_type': training_machine_type, + 'training_total_replica_count': training_total_replica_count, + 'training_accelerator_type': training_accelerator_type, + 'training_accelerator_count': training_accelerator_count, + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': transform_dataflow_max_num_workers, + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'run_evaluation': run_evaluation, + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'dataflow_service_account': dataflow_service_account, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + } + _update_parameters(parameter_values, training_and_eval_parameters) + + fte_params = { + 'dataset_level_custom_transformation_definitions': ( + dataset_level_custom_transformation_definitions + if dataset_level_custom_transformation_definitions + else [] + ), + 'dataset_level_transformations': ( + dataset_level_transformations if dataset_level_transformations else [] + ), + 'run_feature_selection': run_feature_selection, + 'feature_selection_algorithm': feature_selection_algorithm, + 'max_selected_features': max_selected_features, + 'predefined_split_key': predefined_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'tf_auto_transform_features': ( + tf_auto_transform_features if tf_auto_transform_features else {} + ), + 'tf_custom_transformation_definitions': ( + tf_custom_transformation_definitions + if tf_custom_transformation_definitions + else [] + ), + 'tf_transformations_path': tf_transformations_path, + } + _update_parameters(parameter_values, fte_params) + + data_source_and_split_parameters = { + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_staging_full_dataset_id': bigquery_staging_full_dataset_id, + } + _update_parameters(parameter_values, data_source_and_split_parameters) + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), 'xgboost_trainer_pipeline.yaml' + ) + + return pipeline_definition_path, parameter_values + + +def get_xgboost_hyperparameter_tuning_job_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + objective: str, + study_spec_metric_id: str, + study_spec_metric_goal: str, + max_trial_count: int, + parallel_trial_count: int, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + eval_metric: Optional[str] = None, + disable_default_eval_metric: Optional[int] = None, + seed: Optional[int] = None, + seed_per_iteration: Optional[bool] = None, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + run_feature_selection: Optional[bool] = None, + feature_selection_algorithm: Optional[str] = None, + max_selected_features: Optional[int] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + tf_custom_transformation_definitions: Optional[List[Dict[str, Any]]] = None, + tf_transformations_path: Optional[str] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + weight_column: Optional[str] = None, + max_failed_trial_count: Optional[int] = None, + training_machine_type: Optional[str] = None, + training_total_replica_count: Optional[int] = None, + training_accelerator_type: Optional[str] = None, + training_accelerator_count: Optional[int] = None, + study_spec_algorithm: Optional[str] = None, + study_spec_measurement_selection_type: Optional[str] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + run_evaluation: Optional[bool] = None, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, + dataflow_service_account: Optional[str] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: Optional[bool] = None, + encryption_spec_key_name: Optional[str] = None, +): + """Get the XGBoost HyperparameterTuningJob pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + objective: Specifies the learning task and the learning objective. Must be + one of [reg:squarederror, reg:squaredlogerror, + reg:logistic, reg:gamma, reg:tweedie, reg:pseudohubererror, + binary:logistic, multi:softprob]. + study_spec_metric_id: Metric to optimize. For options, please look under + 'eval_metric' at + https://xgboost.readthedocs.io/en/stable/parameter.html#learning-task-parameters. + study_spec_metric_goal: Optimization goal of the metric, possible values: + "MAXIMIZE", "MINIMIZE". + max_trial_count: The desired total number of trials. + parallel_trial_count: The desired number of trials to run in parallel. + study_spec_parameters_override: List of dictionaries representing parameters + to optimize. The dictionary key is the parameter_id, which is passed to + training job as a command line argument, and the dictionary value is the + parameter specification of the metric. + eval_metric: Evaluation metrics for validation data represented as a + comma-separated string. + disable_default_eval_metric: Flag to disable default metric. Set to >0 to + disable. Default to 0. + seed: Random seed. + seed_per_iteration: Seed PRNG determnisticly via iterator number. + dataset_level_custom_transformation_definitions: Dataset-level custom + transformation definitions in string format. + dataset_level_transformations: Dataset-level transformation configuration in + string format. + run_feature_selection: Whether to enable feature selection. + feature_selection_algorithm: Feature selection algorithm. + max_selected_features: Maximum number of features to select. + predefined_split_key: Predefined split key. + stratified_split_key: Stratified split key. + training_fraction: Training fraction. + validation_fraction: Validation fraction. + test_fraction: Test fraction. + tf_auto_transform_features: List of auto transform features in the + comma-separated string format. + tf_custom_transformation_definitions: TF custom transformation definitions + in string format. + tf_transformations_path: Path to TF transformation configuration. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + bigquery_staging_full_dataset_id: The BigQuery staging full dataset id for + storing intermediate tables. + weight_column: The weight column name. + max_failed_trial_count: The number of failed trials that need to be seen + before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides + how many trials must fail before the whole job fails. + training_machine_type: Machine type. + training_total_replica_count: Number of workers. + training_accelerator_type: Accelerator type. + training_accelerator_count: Accelerator count. + study_spec_algorithm: The search algorithm specified for the study. One of + 'ALGORITHM_UNSPECIFIED', 'GRID_SEARCH', or 'RANDOM_SEARCH'. + study_spec_measurement_selection_type: Which measurement to use if/when the + service automatically selects the final measurement from previously + reported intermediate measurements. One of "BEST_MEASUREMENT" or + "LAST_MEASUREMENT". + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + run_evaluation: Whether to run evaluation steps during training. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = {} + if isinstance(tf_auto_transform_features, list): + tf_auto_transform_features = {'auto': tf_auto_transform_features} + + training_and_eval_parameters = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'objective': objective, + 'eval_metric': eval_metric, + 'study_spec_metric_id': study_spec_metric_id, + 'study_spec_metric_goal': study_spec_metric_goal, + 'max_trial_count': max_trial_count, + 'parallel_trial_count': parallel_trial_count, + 'study_spec_parameters_override': ( + study_spec_parameters_override + if study_spec_parameters_override + else [] + ), + 'disable_default_eval_metric': disable_default_eval_metric, + 'seed': seed, + 'seed_per_iteration': seed_per_iteration, + 'weight_column': weight_column, + 'max_failed_trial_count': max_failed_trial_count, + 'training_machine_type': training_machine_type, + 'training_total_replica_count': training_total_replica_count, + 'training_accelerator_type': training_accelerator_type, + 'training_accelerator_count': training_accelerator_count, + 'study_spec_algorithm': study_spec_algorithm, + 'study_spec_measurement_selection_type': ( + study_spec_measurement_selection_type + ), + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': transform_dataflow_max_num_workers, + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'run_evaluation': run_evaluation, + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'dataflow_service_account': dataflow_service_account, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + } + _update_parameters(parameter_values, training_and_eval_parameters) + + fte_params = { + 'dataset_level_custom_transformation_definitions': ( + dataset_level_custom_transformation_definitions + if dataset_level_custom_transformation_definitions + else [] + ), + 'dataset_level_transformations': ( + dataset_level_transformations if dataset_level_transformations else [] + ), + 'run_feature_selection': run_feature_selection, + 'feature_selection_algorithm': feature_selection_algorithm, + 'max_selected_features': max_selected_features, + 'predefined_split_key': predefined_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'tf_auto_transform_features': ( + tf_auto_transform_features if tf_auto_transform_features else {} + ), + 'tf_custom_transformation_definitions': ( + tf_custom_transformation_definitions + if tf_custom_transformation_definitions + else [] + ), + 'tf_transformations_path': tf_transformations_path, + } + _update_parameters(parameter_values, fte_params) + + data_source_and_split_parameters = { + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_staging_full_dataset_id': bigquery_staging_full_dataset_id, + } + _update_parameters(parameter_values, data_source_and_split_parameters) + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'xgboost_hyperparameter_tuning_job_pipeline.yaml', + ) + + return pipeline_definition_path, parameter_values + + +def get_feature_selection_pipeline_and_parameters( + root_dir: str, + project: str, + location: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + dataset_level_custom_transformation_definitions: Optional[ + List[Dict[str, Any]] + ] = None, + dataset_level_transformations: Optional[List[Dict[str, Any]]] = None, + run_feature_selection: Optional[bool] = None, + feature_selection_algorithm: Optional[str] = None, + feature_selection_execution_engine: Optional[ + str + ] = _FEATURE_SELECTION_EXECUTION_ENGINE_BIGQUERY, + max_selected_features: Optional[int] = None, + predefined_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + tf_auto_transform_features: Optional[ + Union[List[str], Dict[str, List[str]]] + ] = None, + weight_column: Optional[str] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + bigquery_staging_full_dataset_id: Optional[str] = None, + dataflow_machine_type: Optional[str] = None, + dataflow_max_num_workers: Optional[int] = None, + dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: Optional[bool] = None, + encryption_spec_key_name: Optional[str] = None, + stage_1_deadline_hours: Optional[float] = None, + stage_2_deadline_hours: Optional[float] = None, +): + """Returns feature transform engine pipeline and formatted parameters.""" + + if isinstance(tf_auto_transform_features, list): + tf_auto_transform_features = {'auto': tf_auto_transform_features} + + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), 'feature_selection_pipeline.yaml' + ) + + parameter_values = { + 'root_dir': root_dir, + 'project': project, + 'location': location, + 'target_column': target_column, + 'weight_column': weight_column, + 'prediction_type': prediction_type, + 'dataset_level_custom_transformation_definitions': ( + dataset_level_custom_transformation_definitions + if dataset_level_custom_transformation_definitions + else [] + ), + 'dataset_level_transformations': ( + dataset_level_transformations if dataset_level_transformations else [] + ), + 'run_feature_selection': run_feature_selection, + 'feature_selection_algorithm': feature_selection_algorithm, + 'feature_selection_execution_engine': feature_selection_execution_engine, + 'max_selected_features': max_selected_features, + 'predefined_split_key': predefined_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'tf_auto_transform_features': tf_auto_transform_features, + 'optimization_objective': optimization_objective, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_staging_full_dataset_id': bigquery_staging_full_dataset_id, + 'dataflow_machine_type': dataflow_machine_type, + 'dataflow_max_num_workers': dataflow_max_num_workers, + 'dataflow_disk_size_gb': dataflow_disk_size_gb, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + 'stage_1_deadline_hours': stage_1_deadline_hours, + 'stage_2_deadline_hours': stage_2_deadline_hours, + } + + parameter_values = { + param: value + for param, value in parameter_values.items() + if value is not None + } + + return pipeline_definition_path, parameter_values diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py new file mode 100644 index 0000000000..2abc66bcc2 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job.py @@ -0,0 +1,213 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Wide and Deep Hyperparameter Tuning component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input + + +@dsl.container_component +def wide_and_deep_hyperparameter_tuning_job( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + study_spec_metric_id: str, + study_spec_metric_goal: str, + study_spec_parameters_override: list, + max_trial_count: int, + parallel_trial_count: int, + instance_baseline: Input[Artifact], + metadata: Input[Artifact], + materialized_train_split: Input[Artifact], + materialized_eval_split: Input[Artifact], + transform_output: Input[Artifact], + training_schema_uri: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + instance_schema_uri: dsl.OutputPath(str), + prediction_schema_uri: dsl.OutputPath(str), + trials: dsl.OutputPath(str), + prediction_docker_uri_output: dsl.OutputPath(str), + execution_metrics: dsl.OutputPath(dict), + weight_column: Optional[str] = '', + enable_profiler: Optional[bool] = False, + cache_data: Optional[str] = 'auto', + seed: Optional[int] = 1, + eval_steps: Optional[int] = 0, + eval_frequency_secs: Optional[int] = 600, + max_failed_trial_count: Optional[int] = 0, + study_spec_algorithm: Optional[str] = 'ALGORITHM_UNSPECIFIED', + study_spec_measurement_selection_type: Optional[str] = 'BEST_MEASUREMENT', + training_machine_spec: Optional[dict] = {'machine_type': 'c2-standard-16'}, + training_disk_spec: Optional[dict] = { + 'boot_disk_type': 'pd-ssd', + 'boot_disk_size_gb': 100, + }, + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Tunes Wide & Deep hyperparameters using Vertex HyperparameterTuningJob API. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + weight_column: The weight column name. + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or negative, it means run evaluation on the whole validation dataset. If set to 0, it means run evaluation for a fixed number of samples. + eval_frequency_secs: Frequency at which evaluation and checkpointing will take place. + study_spec_metric_id: Metric to optimize, possible values: [ 'loss', 'average_loss', 'rmse', 'mae', 'mql', 'accuracy', 'auc', 'precision', 'recall']. + study_spec_metric_goal: Optimization goal of the metric, possible values: "MAXIMIZE", "MINIMIZE". + study_spec_parameters_override: List of dictionaries representing parameters to optimize. The dictionary key is the parameter_id, which is passed to training job as a command line argument, and the dictionary value is the parameter specification of the metric. + max_trial_count: The desired total number of trials. + parallel_trial_count: The desired number of trials to run in parallel. + max_failed_trial_count: The number of failed trials that need to be seen before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides how many trials must fail before the whole job fails. + study_spec_algorithm: The search algorithm specified for the study. One of 'ALGORITHM_UNSPECIFIED', 'GRID_SEARCH', or 'RANDOM_SEARCH'. + study_spec_measurement_selection_type: Which measurement to use if/when the service automatically selects the final measurement from previously reported intermediate measurements. One of "BEST_MEASUREMENT" or "LAST_MEASUREMENT". + training_machine_spec: The training machine spec. See https://cloud.google.com/compute/docs/machine-types for options. + training_disk_spec: The training disk spec. + instance_baseline: The path to a JSON file for baseline values. + metadata: Amount of time in seconds to run the trainer for. + materialized_train_split: The path to the materialized train split. + materialized_eval_split: The path to the materialized validation split. + transform_output: The path to transform output. + training_schema_uri: The path to the training schema. + encryption_spec_key_name: The KMS key name. + + Returns: + gcp_resources: Serialized gcp_resources proto tracking the custom training job. + instance_schema_uri: The path to the instance schema. + prediction_schema_uri: The path to the prediction schema. + trials: The path to the hyperparameter tuning trials + prediction_docker_uri_output: The URI of the prediction container. + execution_metrics: Core metrics in dictionary of hyperparameter tuning job execution. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.hyperparameter_tuning_job.launcher', + ], + args=[ + '--type', + 'HyperparameterTuningJobWithMetrics', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--execution_metrics', + execution_metrics, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "wide-and-deep-hyperparameter-tuning-job-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + '"}, "study_spec": {"metrics": [{"metric_id": "', + study_spec_metric_id, + '", "goal": "', + study_spec_metric_goal, + '"}], "parameters": ', + study_spec_parameters_override, + ', "algorithm": "', + study_spec_algorithm, + '", "measurement_selection_type": "', + study_spec_measurement_selection_type, + '"}, "max_trial_count": ', + max_trial_count, + ', "parallel_trial_count": ', + parallel_trial_count, + ', "max_failed_trial_count": ', + max_failed_trial_count, + ( + ', "trial_job_spec": {"worker_pool_specs":' + ' [{"replica_count":"' + ), + '1', + '", "machine_spec": ', + training_machine_spec, + ', "disk_spec": ', + training_disk_spec, + ', "container_spec": {"image_uri":"', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20231002_0125', + '", "args": ["--target_column=', + target_column, + '", "--weight_column=', + weight_column, + '", "--model_type=', + prediction_type, + '", "--prediction_docker_uri=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125', + '", "--prediction_docker_uri_artifact_path=', + prediction_docker_uri_output, + '", "--baseline_path=', + instance_baseline.uri, + '", "--metadata_path=', + metadata.uri, + '", "--transform_output_path=', + transform_output.uri, + '", "--training_schema_path=', + training_schema_uri.uri, + '", "--instance_schema_path=', + instance_schema_uri, + '", "--prediction_schema_path=', + prediction_schema_uri, + '", "--trials_path=', + trials, + '", "--job_dir=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train",' + ' "--training_data_path=' + ), + materialized_train_split.uri, + '", "--validation_data_path=', + materialized_eval_split.uri, + '", "--enable_profiler=', + enable_profiler, + '", "--cache_data=', + cache_data, + '", "--measurement_selection_type=', + study_spec_measurement_selection_type, + '", "--metric_goal=', + study_spec_metric_goal, + '", "--seed=', + seed, + '", "--eval_steps=', + eval_steps, + '", "--eval_frequency_secs=', + eval_frequency_secs, + '"]}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml new file mode 100644 index 0000000000..f1c88723a8 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_hyperparameter_tuning_job_pipeline.yaml @@ -0,0 +1,4092 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-wide-and-deep-hyperparameter-tuning-job +# Description: The Wide & Deep built-in algorithm HyperparameterTuningJob pipeline. +# Inputs: +# bigquery_staging_full_dataset_id: str [Default: ''] +# cache_data: str [Default: 'auto'] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# enable_profiler: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# eval_frequency_secs: int [Default: 600.0] +# eval_steps: int [Default: 0.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# feature_selection_algorithm: str [Default: 'AMI'] +# location: str +# materialized_examples_format: str [Default: 'tfrecords_gzip'] +# max_failed_trial_count: int [Default: 0.0] +# max_selected_features: int [Default: -1.0] +# max_trial_count: int +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# parallel_trial_count: int +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# root_dir: str +# run_evaluation: bool [Default: False] +# run_feature_selection: bool [Default: False] +# seed: int [Default: 1.0] +# stratified_split_key: str [Default: ''] +# study_spec_algorithm: str [Default: 'ALGORITHM_UNSPECIFIED'] +# study_spec_measurement_selection_type: str [Default: 'BEST_MEASUREMENT'] +# study_spec_metric_goal: str +# study_spec_metric_id: str +# study_spec_parameters_override: list +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# tf_custom_transformation_definitions: list +# tf_transform_execution_engine: str [Default: ''] +# tf_transformations_path: str [Default: ''] +# training_fraction: float [Default: -1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# worker_pool_specs_override: list +# Outputs: +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-infra-validator: + executorLabel: exec-automl-tabular-infra-validator + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + inputDefinitions: + artifacts: + pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + automl-tabular-infra-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator + dependentTasks: + - get-best-hyperparameter-tuning-job-trial + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + taskInfo: + name: automl-tabular-infra-validator + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - bool-identity + - feature-transform-engine + - get-best-hyperparameter-tuning-job-trial + inputs: + artifacts: + pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + parameters: + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: pipelinechannel--dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: pipelinechannel--dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: pipelinechannel--feature_selection_algorithm + location: + componentInputParameter: pipelinechannel--location + materialized_examples_format: + componentInputParameter: pipelinechannel--materialized_examples_format + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + model_type: + runtimeValue: + constant: neural_network + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_feature_selection: + componentInputParameter: pipelinechannel--run_feature_selection + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--tf_auto_transform_features + tf_custom_transformation_definitions: + componentInputParameter: pipelinechannel--tf_custom_transformation_definitions + tf_transform_execution_engine: + componentInputParameter: pipelinechannel--tf_transform_execution_engine + tf_transformations_path: + componentInputParameter: pipelinechannel--tf_transformations_path + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + get-best-hyperparameter-tuning-job-trial: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-best-hyperparameter-tuning-job-trial + dependentTasks: + - wide-and-deep-hyperparameter-tuning-job + inputs: + parameters: + gcp_resources: + taskOutputParameter: + outputParameterKey: gcp_resources + producerTask: wide-and-deep-hyperparameter-tuning-job + instance_schema_uri: + taskOutputParameter: + outputParameterKey: instance_schema_uri + producerTask: wide-and-deep-hyperparameter-tuning-job + prediction_docker_uri: + taskOutputParameter: + outputParameterKey: prediction_docker_uri_output + producerTask: wide-and-deep-hyperparameter-tuning-job + prediction_schema_uri: + taskOutputParameter: + outputParameterKey: prediction_schema_uri + producerTask: wide-and-deep-hyperparameter-tuning-job + study_spec_metric_goal: + componentInputParameter: pipelinechannel--study_spec_metric_goal + trials_dir: + taskOutputParameter: + outputParameterKey: trials + producerTask: wide-and-deep-hyperparameter-tuning-job + taskInfo: + name: get-best-hyperparameter-tuning-job-trial + get-wide-and-deep-study-spec-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-wide-and-deep-study-spec-parameters + inputs: + parameters: + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + taskInfo: + name: get-wide-and-deep-study-spec-parameters + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-tabular-infra-validator + - get-best-hyperparameter-tuning-job-trial + inputs: + artifacts: + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + parse-worker-pool-specs-override: + cachingOptions: + enableCache: true + componentRef: + name: comp-parse-worker-pool-specs-override + inputs: + parameters: + worker_pool_specs_override: + componentInputParameter: pipelinechannel--worker_pool_specs_override + taskInfo: + name: parse-worker-pool-specs-override + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + wide-and-deep-hyperparameter-tuning-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-wide-and-deep-hyperparameter-tuning-job + dependentTasks: + - feature-transform-engine + - get-wide-and-deep-study-spec-parameters + - parse-worker-pool-specs-override + - split-materialized-data + - training-configurator-and-validator + inputs: + artifacts: + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + training_schema_uri: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + parameters: + cache_data: + componentInputParameter: pipelinechannel--cache_data + enable_profiler: + componentInputParameter: pipelinechannel--enable_profiler + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + eval_frequency_secs: + componentInputParameter: pipelinechannel--eval_frequency_secs + eval_steps: + componentInputParameter: pipelinechannel--eval_steps + location: + componentInputParameter: pipelinechannel--location + max_failed_trial_count: + componentInputParameter: pipelinechannel--max_failed_trial_count + max_trial_count: + componentInputParameter: pipelinechannel--max_trial_count + parallel_trial_count: + componentInputParameter: pipelinechannel--parallel_trial_count + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + seed: + componentInputParameter: pipelinechannel--seed + study_spec_algorithm: + componentInputParameter: pipelinechannel--study_spec_algorithm + study_spec_measurement_selection_type: + componentInputParameter: pipelinechannel--study_spec_measurement_selection_type + study_spec_metric_goal: + componentInputParameter: pipelinechannel--study_spec_metric_goal + study_spec_metric_id: + componentInputParameter: pipelinechannel--study_spec_metric_id + study_spec_parameters_override: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-wide-and-deep-study-spec-parameters + target_column: + componentInputParameter: pipelinechannel--target_column + training_disk_spec: + taskOutputParameter: + outputParameterKey: training_disk_spec + producerTask: parse-worker-pool-specs-override + training_machine_spec: + taskOutputParameter: + outputParameterKey: training_machine_spec + producerTask: parse-worker-pool-specs-override + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: wide-and-deep-hyperparameter-tuning-job + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--cache_data: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--dataset_level_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--dataset_level_transformations: + parameterType: LIST + pipelinechannel--enable_profiler: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--eval_frequency_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--eval_steps: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_selection_algorithm: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--materialized_examples_format: + parameterType: STRING + pipelinechannel--max_failed_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--max_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--parallel_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--run_feature_selection: + parameterType: BOOLEAN + pipelinechannel--seed: + parameterType: NUMBER_INTEGER + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--study_spec_algorithm: + parameterType: STRING + pipelinechannel--study_spec_measurement_selection_type: + parameterType: STRING + pipelinechannel--study_spec_metric_goal: + parameterType: STRING + pipelinechannel--study_spec_metric_id: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--tf_auto_transform_features: + parameterType: STRUCT + pipelinechannel--tf_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--tf_transform_execution_engine: + parameterType: STRING + pipelinechannel--tf_transformations_path: + parameterType: STRING + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--worker_pool_specs_override: + parameterType: LIST + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-get-best-hyperparameter-tuning-job-trial: + executorLabel: exec-get-best-hyperparameter-tuning-job-trial + inputDefinitions: + parameters: + gcp_resources: + description: Proto tracking the hyperparameter tuning job. + parameterType: STRING + instance_schema_uri: + defaultValue: '' + description: The instance schema uri. + isOptional: true + parameterType: STRING + prediction_docker_uri: + defaultValue: '' + description: The prediction docker container uri. + isOptional: true + parameterType: STRING + prediction_schema_uri: + defaultValue: '' + description: The prediction schema_uri. + isOptional: true + parameterType: STRING + read_value_from_file: + defaultValue: false + description: If true, read file to get the relevant value. + isOptional: true + parameterType: BOOLEAN + study_spec_metric_goal: + description: 'Optimization goal of the metric, possible values: + + "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + trials_dir: + defaultValue: '' + description: The path to the hyperparameter tuning trials. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-get-wide-and-deep-study-spec-parameters: + executorLabel: exec-get-wide-and-deep-study-spec-parameters + inputDefinitions: + parameters: + study_spec_parameters_override: + description: 'List of dictionaries representing parameters + + to optimize. The dictionary key is the parameter_id, which is passed to + + training job as a command line argument, and the dictionary value is the + + parameter specification of the metric.' + parameterType: LIST + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: An artifact of a model which to upload a new version to. Only + specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: "The unmanaged container model to be uploaded. The Model can\ + \ be passed from an upstream step or imported via a KFP `dsl.importer`.\n\ + :Examples:\n ::\n\n from kfp import dsl\n from google_cloud_pipeline_components.google_cloud_pipeline_components.types\ + \ import artifact_types\n\n importer_spec = dsl.importer(\n artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model',\n\ + \ artifact_class=artifact_types.UnmanagedContainerModel,\n metadata={\n\ + \ 'containerSpec': { 'imageUri':\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod'\n\ + \ }\n })" + isOptional: true + parameters: + description: + defaultValue: '' + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + isOptional: true + parameterType: STRING + display_name: + description: 'The display name of the Model. The name + + can be up to 128 characters long and can be consist of any UTF-8 + + characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model)' + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key spec for a Model. If set, this Model and all sub-resources of this + + Model will be secured by this key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + description: 'Metadata describing the Model''s + + input and output for explanation. Both `explanation_metadata` and `explanation_parameters` + must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata)' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters)' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your model. Label keys and values can be no longer than 64 + + characters (Unicode codepoints), can only contain lowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. See https://goo.gl/xmQnxf for more information and + + examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Optional location to upload this Model to. If + + not set, defaults to `us-central1`.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to upload this Model to. Defaults to the project in + which the PipelineJob is run. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: Artifact tracking the created Model. + parameters: + gcp_resources: + description: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) + which tracks the upload Model's long-running operation. + parameterType: STRING + comp-parse-worker-pool-specs-override: + executorLabel: exec-parse-worker-pool-specs-override + inputDefinitions: + parameters: + worker_pool_specs_override: + description: 'The list of dictionaries for overriding training + + and evaluation worker pool specs.' + parameterType: LIST + outputDefinitions: + parameters: + eval_machine_spec: + description: The eval machine spec. + parameterType: STRUCT + eval_replica_count: + description: The replica count for eval. + parameterType: NUMBER_INTEGER + training_disk_spec: + description: The training disk spec. + parameterType: STRUCT + training_machine_spec: + description: The training machine spec. + parameterType: STRUCT + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + comp-wide-and-deep-hyperparameter-tuning-job: + executorLabel: exec-wide-and-deep-hyperparameter-tuning-job + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to a JSON file for baseline values. + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized validation split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized train split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Amount of time in seconds to run the trainer for. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to transform output. + parameters: + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to + + ''auto'', caching is determined based on the dataset size.' + isOptional: true + parameterType: STRING + enable_profiler: + defaultValue: false + description: 'Enables profiling and saves a trace + + during evaluation.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and + + checkpointing will take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not + + specified or negative, it means run evaluation on the whole validation + + dataset. If set to 0, it means run evaluation for a fixed number of + + samples.' + isOptional: true + parameterType: NUMBER_INTEGER + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + max_failed_trial_count: + defaultValue: 0.0 + description: 'The number of failed trials that + + need to be seen before failing the HyperparameterTuningJob. If set to + 0, + + Vertex AI decides how many trials must fail before the whole job fails.' + isOptional: true + parameterType: NUMBER_INTEGER + max_trial_count: + description: The desired total number of trials. + parameterType: NUMBER_INTEGER + parallel_trial_count: + description: 'The desired number of trials to run + + in parallel.' + parameterType: NUMBER_INTEGER + prediction_type: + description: 'The type of prediction the model is to + + produce. "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + study_spec_algorithm: + defaultValue: ALGORITHM_UNSPECIFIED + description: 'The search algorithm specified for + + the study. One of ''ALGORITHM_UNSPECIFIED'', ''GRID_SEARCH'', or + + ''RANDOM_SEARCH''.' + isOptional: true + parameterType: STRING + study_spec_measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: 'Which measurement + + to use if/when the service automatically selects the final measurement + + from previously reported intermediate measurements. One of + + "BEST_MEASUREMENT" or "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + study_spec_metric_goal: + description: 'Optimization goal of the metric, + + possible values: "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + study_spec_metric_id: + description: 'Metric to optimize, , possible + + values: [ ''loss'', ''average_loss'', ''rmse'', ''mae'', ''mql'', ''accuracy'', + ''auc'', ''precision'', ''recall''].' + parameterType: STRING + study_spec_parameters_override: + description: 'List of dictionaries + + representing parameters to optimize. The dictionary key is the + + parameter_id, which is passed to training job as a command line + + argument, and the dictionary value is the parameter specification of the + + metric.' + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + training_disk_spec: + defaultValue: + boot_disk_size_gb: 100.0 + boot_disk_type: pd-ssd + description: The training disk spec. + isOptional: true + parameterType: STRUCT + training_machine_spec: + defaultValue: + machine_type: c2-standard-16 + description: 'The training machine + + spec. See https://cloud.google.com/compute/docs/machine-types for + + options.' + isOptional: true + parameterType: STRUCT + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + execution_metrics: + description: Core metrics in dictionary of hyperparameter tuning job execution. + parameterType: STRUCT + gcp_resources: + description: Serialized gcp_resources proto tracking the custom training + job. + parameterType: STRING + instance_schema_uri: + description: The path to the instance schema. + parameterType: STRING + prediction_docker_uri_output: + description: The URI of the prediction container. + parameterType: STRING + prediction_schema_uri: + description: The path to the prediction schema. + parameterType: STRING + trials: + description: The path to the hyperparameter tuning trials + parameterType: STRING +deploymentSpec: + executors: + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-infra-validator: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-get-best-hyperparameter-tuning-job-trial: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_best_hyperparameter_tuning_job_trial + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_best_hyperparameter_tuning_job_trial(\n gcp_resources:\ + \ str,\n study_spec_metric_goal: str,\n unmanaged_container_model:\ + \ dsl.Output[dsl.Artifact],\n trials_dir: str = '',\n instance_schema_uri:\ + \ str = '',\n prediction_schema_uri: str = '',\n prediction_docker_uri:\ + \ str = '',\n read_value_from_file: bool = False,\n):\n \"\"\"Gets best\ + \ HyperparameterTuningJob trial.\n\n Args:\n gcp_resources: Proto tracking\ + \ the hyperparameter tuning job.\n study_spec_metric_goal: Optimization\ + \ goal of the metric, possible values:\n \"MAXIMIZE\", \"MINIMIZE\"\ + .\n unmanaged_container_model: The unmanaged model.\n trials_dir:\ + \ The path to the hyperparameter tuning trials.\n instance_schema_uri:\ + \ The instance schema uri.\n prediction_schema_uri: The prediction schema_uri.\n\ + \ prediction_docker_uri: The prediction docker container uri.\n read_value_from_file:\ + \ If true, read file to get the relevant value.\n\n Raises:\n RuntimeError:\ + \ If there are multiple metrics.\n \"\"\"\n\n import os\n import json\n\ + \ from google.api_core.retry import Retry\n from google.cloud import aiplatform_v1beta1\ + \ as aip\n import tensorflow as tf\n\n # If path to file with value is\ + \ provided, read the file before continuing.\n if read_value_from_file:\n\ + \ with tf.io.gfile.GFile(trials_dir, 'r') as f:\n trials_dir = f.read()\n\ + \ with tf.io.gfile.GFile(instance_schema_uri, 'r') as f:\n instance_schema_uri\ + \ = f.read()\n with tf.io.gfile.GFile(prediction_schema_uri, 'r') as\ + \ f:\n prediction_schema_uri = f.read()\n with tf.io.gfile.GFile(prediction_docker_uri,\ + \ 'r') as f:\n prediction_docker_uri = f.read()\n\n api_endpoint_suffix\ + \ = '-aiplatform.googleapis.com'\n gcp_resources_json = json.loads(gcp_resources)\n\ + \ resource = gcp_resources_json['resources'][0]\n\n uri_key = 'resource_uri'\n\ + \ if uri_key not in resource:\n uri_key = 'resourceUri'\n\n gcp_resources_split\ + \ = resource[uri_key].partition('projects')\n resource_name = gcp_resources_split[1]\ + \ + gcp_resources_split[2]\n prefix_str = gcp_resources_split[0]\n prefix_str\ + \ = prefix_str[: prefix_str.find(api_endpoint_suffix)]\n api_endpoint =\ + \ (\n prefix_str[(prefix_str.rfind('//') + 2) :] + api_endpoint_suffix\n\ + \ )\n\n job_client = aip.JobServiceClient(\n client_options={'api_endpoint':\ + \ api_endpoint}\n )\n response = job_client.get_hyperparameter_tuning_job(\n\ + \ name=resource_name,\n retry=Retry(initial=10.0, maximum=60.0,\ + \ deadline=10.0 * 60.0),\n )\n\n # Get best trial\n trials_list = []\n\ + \ for trial in response.trials:\n if trial.final_measurement:\n \ + \ trials_list.append({\n 'id': trial.id,\n 'objective_value':\ + \ trial.final_measurement.metrics[0].value,\n })\n\n if study_spec_metric_goal\ + \ == 'MAXIMIZE':\n best_fn = max\n elif study_spec_metric_goal == 'MINIMIZE':\n\ + \ best_fn = min\n else:\n raise ValueError(\n f'Unexpected\ + \ study spec metric goal: {study_spec_metric_goal}'\n )\n\n best_trial\ + \ = best_fn(trials_list, key=lambda trial: trial['objective_value'])\n\n\ + \ # Build unmanaged_container_model\n unmanaged_container_model.metadata['containerSpec']\ + \ = {\n 'imageUri': prediction_docker_uri,\n 'healthRoute': '/health',\n\ + \ 'predictRoute': '/predict',\n }\n unmanaged_container_model.metadata['predictSchemata']\ + \ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\ + \ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\ + \ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-wide-and-deep-study-spec-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_wide_and_deep_study_spec_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_wide_and_deep_study_spec_parameters(\n study_spec_parameters_override:\ + \ list # Required for KFP validation; pylint:disable=g-bare-generic\n)\ + \ -> list: # Required by KFP; pylint:disable=g-bare-generic\n \"\"\"Get\ + \ study_spec_parameters for a Wide & Deep hyperparameter tuning job.\n\n\ + \ Args:\n study_spec_parameters_override: List of dictionaries representing\ + \ parameters\n to optimize. The dictionary key is the parameter_id,\ + \ which is passed to\n training job as a command line argument, and\ + \ the dictionary value is the\n parameter specification of the metric.\n\ + \n Returns:\n List of final Vizier study_spec_parameters of type ParameterSpec.\n\ + \ \"\"\"\n default_params = [\n {\n 'parameter_id': 'max_steps',\n\ + \ 'discrete_value_spec': {\n 'values': [5000, 10000,\ + \ 20000, 30000, 40000, 50000]\n },\n },\n {\n \ + \ 'parameter_id': 'max_train_secs',\n 'discrete_value_spec':\ + \ {'values': [-1]},\n },\n {\n 'parameter_id': 'learning_rate',\n\ + \ 'double_value_spec': {'min_value': 0.0001, 'max_value': 0.0005},\n\ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n },\n {\n \ + \ 'parameter_id': 'optimizer_type',\n 'categorical_value_spec':\ + \ {'values': ['adam', 'ftrl', 'sgd']},\n },\n {\n 'parameter_id':\ + \ 'l1_regularization_strength',\n 'discrete_value_spec': {'values':\ + \ [0, 0.01, 0.02]},\n },\n {\n 'parameter_id': 'l2_regularization_strength',\n\ + \ 'discrete_value_spec': {'values': [0, 0.01, 0.02]},\n },\n\ + \ {\n 'parameter_id': 'l2_shrinkage_regularization_strength',\n\ + \ 'discrete_value_spec': {'values': [0, 0.01, 0.02]},\n },\n\ + \ {\n 'parameter_id': 'beta_1',\n 'discrete_value_spec':\ + \ {'values': [0.7, 0.8, 0.9]},\n },\n {\n 'parameter_id':\ + \ 'beta_2',\n 'discrete_value_spec': {'values': [0.8, 0.9, 0.999]},\n\ + \ },\n {\n 'parameter_id': 'hidden_units',\n \ + \ 'categorical_value_spec': {'values': ['30,30,30']},\n },\n \ + \ {\n 'parameter_id': 'use_wide',\n 'categorical_value_spec':\ + \ {'values': ['true', 'false']},\n },\n {\n 'parameter_id':\ + \ 'embed_categories',\n 'categorical_value_spec': {'values': ['true',\ + \ 'false']},\n },\n {\n 'parameter_id': 'dnn_dropout',\n\ + \ 'discrete_value_spec': {'values': [0, 0.1, 0.2]},\n },\n\ + \ {\n 'parameter_id': 'dnn_learning_rate',\n 'double_value_spec':\ + \ {'min_value': 0.0001, 'max_value': 0.0005},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n {\n 'parameter_id': 'dnn_optimizer_type',\n \ + \ 'categorical_value_spec': {'values': ['adam', 'ftrl', 'sgd']},\n\ + \ },\n {\n 'parameter_id': 'dnn_l1_regularization_strength',\n\ + \ 'discrete_value_spec': {'values': [0, 0.01, 0.02]},\n },\n\ + \ {\n 'parameter_id': 'dnn_l2_regularization_strength',\n\ + \ 'discrete_value_spec': {'values': [0, 0.01, 0.02]},\n },\n\ + \ {\n 'parameter_id': 'dnn_l2_shrinkage_regularization_strength',\n\ + \ 'discrete_value_spec': {'values': [0, 0.01, 0.02]},\n },\n\ + \ {\n 'parameter_id': 'dnn_beta_1',\n 'discrete_value_spec':\ + \ {'values': [0.7, 0.8, 0.9]},\n },\n {\n 'parameter_id':\ + \ 'dnn_beta_2',\n 'discrete_value_spec': {'values': [0.8, 0.9,\ + \ 0.999]},\n },\n {\n 'parameter_id': 'batch_size',\n\ + \ 'discrete_value_spec': {'values': [1024, 2048, 4096, 8192, 16384]},\n\ + \ },\n ]\n # pylint:disable=g-import-not-at-top,redefined-outer-name\n\ + \ import warnings\n # pylint:enable=g-import-not-at-top,redefined-outer-name\n\ + \n override_params = {}\n for param in study_spec_parameters_override:\n\ + \ override_params[param['parameter_id']] = param\n\n study_spec_parameters\ + \ = []\n for param in default_params:\n study_spec_parameters.append(\n\ + \ override_params.get(param['parameter_id'], param)\n )\n\n extra_overrides\ + \ = set(override_params) - set(\n p['parameter_id'] for p in default_params\n\ + \ )\n if extra_overrides:\n extra_override_str = ', '.join(extra_overrides)\n\ + \ warnings.warn(\n f'The overrides {extra_override_str} were not\ + \ found in the params and '\n 'will be ignored.'\n )\n\n return\ + \ study_spec_parameters\n\n" + image: python:3.7 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"pipeline_job\": + \"", "projects/{{$.inputs.parameters[''project'']}}/locations/{{$.inputs.parameters[''location'']}}/pipelineJobs/{{$.pipeline_job_uuid}}", + "\"", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.model.upload_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-parse-worker-pool-specs-override: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _parse_worker_pool_specs_override + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _parse_worker_pool_specs_override(\n worker_pool_specs_override:\ + \ list, # pylint:disable=g-bare-generic\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('training_machine_spec', dict), # pylint:disable=g-bare-generic\n\ + \ ('training_disk_spec', dict),\n ('eval_machine_spec', dict),\ + \ # pylint:disable=g-bare-generic\n ('eval_replica_count', int),\n\ + \ ],\n):\n \"\"\"Parses worker_pool_specs_override and returns training\ + \ and evaluation machine specifications.\n\n Args:\n worker_pool_specs_override:\ + \ The list of dictionaries for overriding training\n and evaluation\ + \ worker pool specs.\n\n Returns:\n training_machine_spec: The training\ + \ machine spec.\n training_disk_spec: The training disk spec.\n \ + \ eval_machine_spec: The eval machine spec.\n eval_replica_count:\ + \ The replica count for eval.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n training_machine_spec = {'machine_type': 'c2-standard-16'}\n training_disk_spec\ + \ = {'boot_disk_type': 'pd-ssd', 'boot_disk_size_gb': 100}\n eval_machine_spec\ + \ = {'machine_type': 'c2-standard-8'}\n eval_replica_count = 1\n\n if\ + \ worker_pool_specs_override:\n if len(worker_pool_specs_override) >=\ + \ 1 and isinstance(\n worker_pool_specs_override[0], dict\n ):\n\ + \ training_machine_spec = worker_pool_specs_override[0].get(\n \ + \ 'machine_spec', training_machine_spec\n )\n training_disk_spec\ + \ = worker_pool_specs_override[0].get(\n 'disk_spec', training_disk_spec\n\ + \ )\n if len(worker_pool_specs_override) == 4 and isinstance(\n\ + \ worker_pool_specs_override[3], dict\n ):\n eval_machine_spec\ + \ = worker_pool_specs_override[3].get(\n 'machine_spec', eval_machine_spec\n\ + \ )\n eval_replica_count = worker_pool_specs_override[3].get(\n\ + \ 'replica_count', eval_replica_count\n )\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'training_machine_spec',\n \ + \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ + \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ + \ eval_machine_spec,\n eval_replica_count,\n )\n\n" + image: python:3.7 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + exec-wide-and-deep-hyperparameter-tuning-job: + container: + args: + - --type + - HyperparameterTuningJobWithMetrics + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --execution_metrics + - '{{$.outputs.parameters[''execution_metrics''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"wide-and-deep-hyperparameter-tuning-job-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"study_spec\": {\"metrics\": [{\"metric_id\": \"", "{{$.inputs.parameters[''study_spec_metric_id'']}}", + "\", \"goal\": \"", "{{$.inputs.parameters[''study_spec_metric_goal'']}}", + "\"}], \"parameters\": ", "{{$.inputs.parameters[''study_spec_parameters_override'']}}", + ", \"algorithm\": \"", "{{$.inputs.parameters[''study_spec_algorithm'']}}", + "\", \"measurement_selection_type\": \"", "{{$.inputs.parameters[''study_spec_measurement_selection_type'']}}", + "\"}, \"max_trial_count\": ", "{{$.inputs.parameters[''max_trial_count'']}}", + ", \"parallel_trial_count\": ", "{{$.inputs.parameters[''parallel_trial_count'']}}", + ", \"max_failed_trial_count\": ", "{{$.inputs.parameters[''max_failed_trial_count'']}}", + ", \"trial_job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", + "1", "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", + ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20231002_0125", + "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", + "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", + "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--prediction_docker_uri_artifact_path=", "{{$.outputs.parameters[''prediction_docker_uri_output''].output_file}}", + "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_schema_path=", "{{$.inputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--instance_schema_path=", "{{$.outputs.parameters[''instance_schema_uri''].output_file}}", + "\", \"--prediction_schema_path=", "{{$.outputs.parameters[''prediction_schema_uri''].output_file}}", + "\", \"--trials_path=", "{{$.outputs.parameters[''trials''].output_file}}", + "\", \"--job_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--training_data_path=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--validation_data_path=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--enable_profiler=", "{{$.inputs.parameters[''enable_profiler'']}}", + "\", \"--cache_data=", "{{$.inputs.parameters[''cache_data'']}}", "\", \"--measurement_selection_type=", + "{{$.inputs.parameters[''study_spec_measurement_selection_type'']}}", "\", + \"--metric_goal=", "{{$.inputs.parameters[''study_spec_metric_goal'']}}", + "\", \"--seed=", "{{$.inputs.parameters[''seed'']}}", "\", \"--eval_steps=", + "{{$.inputs.parameters[''eval_steps'']}}", "\", \"--eval_frequency_secs=", + "{{$.inputs.parameters[''eval_frequency_secs'']}}", "\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.hyperparameter_tuning_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 +pipelineInfo: + description: The Wide & Deep built-in algorithm HyperparameterTuningJob pipeline. + name: automl-tabular-wide-and-deep-hyperparameter-tuning-job +root: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + pipelinechannel--cache_data: + componentInputParameter: cache_data + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + pipelinechannel--dataset_level_transformations: + componentInputParameter: dataset_level_transformations + pipelinechannel--enable_profiler: + componentInputParameter: enable_profiler + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--eval_frequency_secs: + componentInputParameter: eval_frequency_secs + pipelinechannel--eval_steps: + componentInputParameter: eval_steps + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--materialized_examples_format: + componentInputParameter: materialized_examples_format + pipelinechannel--max_failed_trial_count: + componentInputParameter: max_failed_trial_count + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--max_trial_count: + componentInputParameter: max_trial_count + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--parallel_trial_count: + componentInputParameter: parallel_trial_count + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--prediction_type: + componentInputParameter: prediction_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--run_feature_selection: + componentInputParameter: run_feature_selection + pipelinechannel--seed: + componentInputParameter: seed + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--study_spec_algorithm: + componentInputParameter: study_spec_algorithm + pipelinechannel--study_spec_measurement_selection_type: + componentInputParameter: study_spec_measurement_selection_type + pipelinechannel--study_spec_metric_goal: + componentInputParameter: study_spec_metric_goal + pipelinechannel--study_spec_metric_id: + componentInputParameter: study_spec_metric_id + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + pipelinechannel--tf_custom_transformation_definitions: + componentInputParameter: tf_custom_transformation_definitions + pipelinechannel--tf_transform_execution_engine: + componentInputParameter: tf_transform_execution_engine + pipelinechannel--tf_transformations_path: + componentInputParameter: tf_transformations_path + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--worker_pool_specs_override: + componentInputParameter: worker_pool_specs_override + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Parent model if this model is uploaded as a version. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + bigquery_staging_full_dataset_id: + defaultValue: '' + description: Staging directory for BigQuery tables. + isOptional: true + parameterType: STRING + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to ''auto'', caching is + + determined based on the dataset size.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + description: 'Dataset-level custom + + transformation definitions in string format.' + isOptional: true + parameterType: LIST + dataset_level_transformations: + description: 'Dataset-level transformation configuration in + + string format.' + isOptional: true + parameterType: LIST + enable_profiler: + defaultValue: false + description: Enables profiling and saves a trace during evaluation. + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and checkpointing will + + take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not specified or + + negative, it means run evaluation on the whole validation dataset. If set + + to 0, it means run evaluation for a fixed number of samples.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_selection_algorithm: + defaultValue: AMI + description: Feature selection algorithm. + isOptional: true + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: The format for the materialized examples. + isOptional: true + parameterType: STRING + max_failed_trial_count: + defaultValue: 0.0 + description: 'The number of failed trials that need to be seen + + before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides + + how many trials must fail before the whole job fails.' + isOptional: true + parameterType: NUMBER_INTEGER + max_selected_features: + defaultValue: -1.0 + description: Maximum number of features to select. + isOptional: true + parameterType: NUMBER_INTEGER + max_trial_count: + description: The desired total number of trials. + parameterType: NUMBER_INTEGER + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + parallel_trial_count: + description: The desired number of trials to run in parallel. + parameterType: NUMBER_INTEGER + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to produce. + + "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: Whether to enable feature selection. + isOptional: true + parameterType: BOOLEAN + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + study_spec_algorithm: + defaultValue: ALGORITHM_UNSPECIFIED + description: 'The search algorithm specified for the study. One of + + ''ALGORITHM_UNSPECIFIED'', ''GRID_SEARCH'', or ''RANDOM_SEARCH''.' + isOptional: true + parameterType: STRING + study_spec_measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: ' Which measurement to use if/when the + + service automatically selects the final measurement from previously + + reported intermediate measurements. One of "BEST_MEASUREMENT" or + + "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + study_spec_metric_goal: + description: 'Optimization goal of the metric, possible values: + + "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + study_spec_metric_id: + description: 'Metric to optimize, possible values: [ ''loss'', + + ''average_loss'', ''rmse'', ''mae'', ''mql'', ''accuracy'', ''auc'', ''precision'', + + ''recall''].' + parameterType: STRING + study_spec_parameters_override: + description: 'List of dictionaries representing parameters + + to optimize. The dictionary key is the parameter_id, which is passed to + + training job as a command line argument, and the dictionary value is the + + parameter specification of the metric.' + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: Test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + description: List of auto transform features. + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + description: 'TF custom transformation definitions + + in string format.' + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: '' + description: 'Execution engine to run TF-based + + transformations. Currently supports "dataflow" or "bigquery"' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: Path to TF transformation configuration. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + validation_fraction: + defaultValue: -1.0 + description: Validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + worker_pool_specs_override: + description: 'The dictionary for overriding training and + + evaluation worker pool specs. The dictionary should be of format + + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py new file mode 100644 index 0000000000..ebee0eff06 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer.py @@ -0,0 +1,250 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Wide and Deep Trainer component spec.""" + +from typing import Optional + +from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.container_component +def wide_and_deep_trainer( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + learning_rate: float, + dnn_learning_rate: float, + instance_baseline: Input[Artifact], + metadata: Input[Artifact], + materialized_train_split: Input[Artifact], + materialized_eval_split: Input[Artifact], + transform_output: Input[Artifact], + training_schema_uri: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + unmanaged_container_model: Output[UnmanagedContainerModel], # pylint: disable=unused-argument + weight_column: Optional[str] = '', + max_steps: Optional[int] = -1, + max_train_secs: Optional[int] = -1, + optimizer_type: Optional[str] = 'adam', + l1_regularization_strength: Optional[float] = 0, + l2_regularization_strength: Optional[float] = 0, + l2_shrinkage_regularization_strength: Optional[float] = 0, + beta_1: Optional[float] = 0.9, + beta_2: Optional[float] = 0.999, + hidden_units: Optional[str] = '30,30,30', + use_wide: Optional[bool] = True, + embed_categories: Optional[bool] = True, + dnn_dropout: Optional[float] = 0, + dnn_optimizer_type: Optional[str] = 'ftrl', + dnn_l1_regularization_strength: Optional[float] = 0, + dnn_l2_regularization_strength: Optional[float] = 0, + dnn_l2_shrinkage_regularization_strength: Optional[float] = 0, + dnn_beta_1: Optional[float] = 0.9, + dnn_beta_2: Optional[float] = 0.999, + enable_profiler: Optional[bool] = False, + cache_data: Optional[str] = 'auto', + seed: Optional[int] = 1, + eval_steps: Optional[int] = 0, + batch_size: Optional[int] = 100, + measurement_selection_type: Optional[str] = 'BEST_MEASUREMENT', + optimization_metric: Optional[str] = '', + eval_frequency_secs: Optional[int] = 600, + training_machine_spec: Optional[dict] = {'machine_type': 'c2-standard-16'}, + training_disk_spec: Optional[dict] = { + 'boot_disk_type': 'pd-ssd', + 'boot_disk_size_gb': 100, + }, + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Trains a Wide & Deep model using Vertex CustomJob API. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + weight_column: The weight column name. + max_steps: Number of steps to run the trainer for. + max_train_secs: Amount of time in seconds to run the trainer for. + learning_rate: The learning rate used by the linear optimizer. + optimizer_type: The type of optimizer to use. Choices are "adam", "ftrl" and "sgd" for the Adam, FTRL, and Gradient Descent Optimizers, respectively. + l1_regularization_strength: L1 regularization strength for optimizer_type="ftrl". + l2_regularization_strength: L2 regularization strength for optimizer_type="ftrl" + l2_shrinkage_regularization_strength: L2 shrinkage regularization strength for optimizer_type="ftrl". + beta_1: Beta 1 value for optimizer_type="adam". + beta_2: Beta 2 value for optimizer_type="adam". + hidden_units: Hidden layer sizes to use for DNN feature columns, provided in comma-separated layers. + use_wide: If set to true, the categorical columns will be used in the wide part of the DNN model. + embed_categories: If set to true, the categorical columns will be used embedded and used in the deep part of the model. Embedding size is the square root of the column cardinality. + dnn_dropout: The probability we will drop out a given coordinate. + dnn_learning_rate: The learning rate for training the deep part of the model. + dnn_optimizer_type: The type of optimizer to use for the deep part of the model. Choices are "adam", "ftrl" and "sgd". for the Adam, FTRL, and Gradient Descent Optimizers, respectively. + dnn_l1_regularization_strength: L1 regularization strength for dnn_optimizer_type="ftrl". + dnn_l2_regularization_strength: L2 regularization strength for dnn_optimizer_type="ftrl". + dnn_l2_shrinkage_regularization_strength: L2 shrinkage regularization strength for dnn_optimizer_type="ftrl". + dnn_beta_1: Beta 1 value for dnn_optimizer_type="adam". + dnn_beta_2: Beta 2 value for dnn_optimizer_type="adam". + enable_profiler: Enables profiling and saves a trace during evaluation. + cache_data: Whether to cache data or not. If set to 'auto', caching is determined based on the dataset size. + seed: Seed to be used for this run. + eval_steps: Number of steps to run evaluation for. If not specified or negative, it means run evaluation on the whole validation dataset. If set to 0, it means run evaluation for a fixed number of samples. + batch_size: Batch size for training. + measurement_selection_type: Which measurement to use if/when the service automatically selects the final measurement from previously reported intermediate measurements. One of "BEST_MEASUREMENT" or "LAST_MEASUREMENT". + optimization_metric: Optimization metric used for `measurement_selection_type`. Default is "rmse" for regression and "auc" for classification. + eval_frequency_secs: Frequency at which evaluation and checkpointing will take place. + training_machine_spec: The training machine spec. See https://cloud.google.com/compute/docs/machine-types for options. + training_disk_spec: The training disk spec. + instance_baseline: The path to a JSON file for baseline values. + metadata: Amount of time in seconds to run the trainer for. + materialized_train_split: The path to the materialized train split. + materialized_eval_split: The path to the materialized validation split. + transform_output: The path to transform output. + training_schema_uri: The path to the training schema. + encryption_spec_key_name: The KMS key name. + + Returns: + gcp_resources: Serialized gcp_resources proto tracking the custom training job. + unmanaged_container_model: The UnmanagedContainerModel artifact. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "wide-and-deep-trainer-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":"', + '1', + '", "machine_spec": ', + training_machine_spec, + ', "disk_spec": ', + training_disk_spec, + ', "container_spec": {"image_uri":"', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20231002_0125', + '", "args": ["--target_column=', + target_column, + '", "--weight_column=', + weight_column, + '", "--model_type=', + prediction_type, + '", "--prediction_docker_uri=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125', + '", "--baseline_path=', + instance_baseline.uri, + '", "--metadata_path=', + metadata.uri, + '", "--transform_output_path=', + transform_output.uri, + '", "--training_schema_path=', + training_schema_uri.uri, + '", "--job_dir=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train",' + ' "--training_data_path=' + ), + materialized_train_split.uri, + '", "--validation_data_path=', + materialized_eval_split.uri, + '", "--max_steps=', + max_steps, + '", "--max_train_secs=', + max_train_secs, + '", "--learning_rate=', + learning_rate, + '", "--optimizer_type=', + optimizer_type, + '", "--l1_regularization_strength=', + l1_regularization_strength, + '", "--l2_regularization_strength=', + l2_regularization_strength, + '", "--l2_shrinkage_regularization_strength=', + l2_shrinkage_regularization_strength, + '", "--beta_1=', + beta_1, + '", "--beta_2=', + beta_2, + '", "--hidden_units=', + hidden_units, + '", "--use_wide=', + use_wide, + '", "--embed_categories=', + embed_categories, + '", "--dnn_dropout=', + dnn_dropout, + '", "--dnn_learning_rate=', + dnn_learning_rate, + '", "--dnn_optimizer_type=', + dnn_optimizer_type, + '", "--dnn_l1_regularization_strength=', + dnn_l1_regularization_strength, + '", "--dnn_l2_regularization_strength=', + dnn_l2_regularization_strength, + '", "--dnn_l2_shrinkage_regularization_strength=', + dnn_l2_shrinkage_regularization_strength, + '", "--dnn_beta_1=', + dnn_beta_1, + '", "--dnn_beta_2=', + dnn_beta_2, + '", "--enable_profiler=', + enable_profiler, + '", "--cache_data=', + cache_data, + '", "--seed=', + seed, + '", "--eval_steps=', + eval_steps, + '", "--batch_size=', + batch_size, + '", "--measurement_selection_type=', + measurement_selection_type, + '", "--optimization_metric=', + optimization_metric, + '", "--eval_frequency_secs=', + eval_frequency_secs, + '", "--executor_input={{$.json_escape[1]}}"]}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml new file mode 100644 index 0000000000..c4e5f204e1 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/wide_and_deep_trainer_pipeline.yaml @@ -0,0 +1,4128 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-wide-and-deep-trainer +# Description: Train a model using the Tabular Workflow for Wide & Deep pipelines. +# Wide & Deep jointly trains wide linear models and deep neural networks. It +# combines the benefits of memorization and generalization. +# Inputs: +# batch_size: int [Default: 100.0] +# beta_1: float [Default: 0.9] +# beta_2: float [Default: 0.999] +# bigquery_staging_full_dataset_id: str [Default: ''] +# cache_data: str [Default: 'auto'] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# dnn_beta_1: float [Default: 0.9] +# dnn_beta_2: float [Default: 0.999] +# dnn_dropout: float [Default: 0.0] +# dnn_l1_regularization_strength: float [Default: 0.0] +# dnn_l2_regularization_strength: float [Default: 0.0] +# dnn_l2_shrinkage_regularization_strength: float [Default: 0.0] +# dnn_learning_rate: float +# dnn_optimizer_type: str [Default: 'adam'] +# embed_categories: bool [Default: True] +# enable_profiler: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# eval_frequency_secs: int [Default: 600.0] +# eval_steps: int [Default: 0.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# feature_selection_algorithm: str [Default: 'AMI'] +# hidden_units: str [Default: '30,30,30'] +# l1_regularization_strength: float [Default: 0.0] +# l2_regularization_strength: float [Default: 0.0] +# l2_shrinkage_regularization_strength: float [Default: 0.0] +# learning_rate: float +# location: str +# materialized_examples_format: str [Default: 'tfrecords_gzip'] +# max_selected_features: int [Default: -1.0] +# max_steps: int [Default: -1.0] +# max_train_secs: int [Default: -1.0] +# measurement_selection_type: str [Default: 'BEST_MEASUREMENT'] +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# optimization_metric: str [Default: ''] +# optimizer_type: str [Default: 'adam'] +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# root_dir: str +# run_evaluation: bool [Default: False] +# run_feature_selection: bool [Default: False] +# seed: int [Default: 1.0] +# stratified_split_key: str [Default: ''] +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# tf_custom_transformation_definitions: list +# tf_transform_execution_engine: str [Default: ''] +# tf_transformations_path: str [Default: ''] +# training_fraction: float [Default: -1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# use_wide: bool [Default: True] +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# worker_pool_specs_override: list +# Outputs: +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-infra-validator: + executorLabel: exec-automl-tabular-infra-validator + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--wide-and-deep-trainer-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + inputDefinitions: + artifacts: + pipelinechannel--wide-and-deep-trainer-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + automl-tabular-infra-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator + dependentTasks: + - wide-and-deep-trainer + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: wide-and-deep-trainer + taskInfo: + name: automl-tabular-infra-validator + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - bool-identity + - feature-transform-engine + - wide-and-deep-trainer + inputs: + artifacts: + pipelinechannel--wide-and-deep-trainer-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: wide-and-deep-trainer + parameters: + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: pipelinechannel--dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: pipelinechannel--dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: pipelinechannel--feature_selection_algorithm + location: + componentInputParameter: pipelinechannel--location + materialized_examples_format: + componentInputParameter: pipelinechannel--materialized_examples_format + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + model_type: + runtimeValue: + constant: neural_network + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_feature_selection: + componentInputParameter: pipelinechannel--run_feature_selection + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--tf_auto_transform_features + tf_custom_transformation_definitions: + componentInputParameter: pipelinechannel--tf_custom_transformation_definitions + tf_transform_execution_engine: + componentInputParameter: pipelinechannel--tf_transform_execution_engine + tf_transformations_path: + componentInputParameter: pipelinechannel--tf_transformations_path + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-tabular-infra-validator + - wide-and-deep-trainer + inputs: + artifacts: + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: wide-and-deep-trainer + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + parse-worker-pool-specs-override: + cachingOptions: + enableCache: true + componentRef: + name: comp-parse-worker-pool-specs-override + inputs: + parameters: + worker_pool_specs_override: + componentInputParameter: pipelinechannel--worker_pool_specs_override + taskInfo: + name: parse-worker-pool-specs-override + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + wide-and-deep-trainer: + cachingOptions: + enableCache: true + componentRef: + name: comp-wide-and-deep-trainer + dependentTasks: + - feature-transform-engine + - parse-worker-pool-specs-override + - split-materialized-data + - training-configurator-and-validator + inputs: + artifacts: + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: training-configurator-and-validator + training_schema_uri: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + parameters: + batch_size: + componentInputParameter: pipelinechannel--batch_size + beta_1: + componentInputParameter: pipelinechannel--beta_1 + beta_2: + componentInputParameter: pipelinechannel--beta_2 + cache_data: + componentInputParameter: pipelinechannel--cache_data + dnn_beta_1: + componentInputParameter: pipelinechannel--dnn_beta_1 + dnn_beta_2: + componentInputParameter: pipelinechannel--dnn_beta_2 + dnn_dropout: + componentInputParameter: pipelinechannel--dnn_dropout + dnn_l1_regularization_strength: + componentInputParameter: pipelinechannel--dnn_l1_regularization_strength + dnn_l2_regularization_strength: + componentInputParameter: pipelinechannel--dnn_l2_regularization_strength + dnn_l2_shrinkage_regularization_strength: + componentInputParameter: pipelinechannel--dnn_l2_shrinkage_regularization_strength + dnn_learning_rate: + componentInputParameter: pipelinechannel--dnn_learning_rate + dnn_optimizer_type: + componentInputParameter: pipelinechannel--dnn_optimizer_type + embed_categories: + componentInputParameter: pipelinechannel--embed_categories + enable_profiler: + componentInputParameter: pipelinechannel--enable_profiler + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + eval_frequency_secs: + componentInputParameter: pipelinechannel--eval_frequency_secs + eval_steps: + componentInputParameter: pipelinechannel--eval_steps + hidden_units: + componentInputParameter: pipelinechannel--hidden_units + l1_regularization_strength: + componentInputParameter: pipelinechannel--l1_regularization_strength + l2_regularization_strength: + componentInputParameter: pipelinechannel--l2_regularization_strength + l2_shrinkage_regularization_strength: + componentInputParameter: pipelinechannel--l2_shrinkage_regularization_strength + learning_rate: + componentInputParameter: pipelinechannel--learning_rate + location: + componentInputParameter: pipelinechannel--location + max_steps: + componentInputParameter: pipelinechannel--max_steps + max_train_secs: + componentInputParameter: pipelinechannel--max_train_secs + measurement_selection_type: + componentInputParameter: pipelinechannel--measurement_selection_type + optimization_metric: + componentInputParameter: pipelinechannel--optimization_metric + optimizer_type: + componentInputParameter: pipelinechannel--optimizer_type + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + seed: + componentInputParameter: pipelinechannel--seed + target_column: + componentInputParameter: pipelinechannel--target_column + training_disk_spec: + taskOutputParameter: + outputParameterKey: training_disk_spec + producerTask: parse-worker-pool-specs-override + training_machine_spec: + taskOutputParameter: + outputParameterKey: training_machine_spec + producerTask: parse-worker-pool-specs-override + use_wide: + componentInputParameter: pipelinechannel--use_wide + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: wide-and-deep-trainer + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--batch_size: + parameterType: NUMBER_INTEGER + pipelinechannel--beta_1: + parameterType: NUMBER_DOUBLE + pipelinechannel--beta_2: + parameterType: NUMBER_DOUBLE + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--cache_data: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--dataset_level_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--dataset_level_transformations: + parameterType: LIST + pipelinechannel--dnn_beta_1: + parameterType: NUMBER_DOUBLE + pipelinechannel--dnn_beta_2: + parameterType: NUMBER_DOUBLE + pipelinechannel--dnn_dropout: + parameterType: NUMBER_DOUBLE + pipelinechannel--dnn_l1_regularization_strength: + parameterType: NUMBER_DOUBLE + pipelinechannel--dnn_l2_regularization_strength: + parameterType: NUMBER_DOUBLE + pipelinechannel--dnn_l2_shrinkage_regularization_strength: + parameterType: NUMBER_DOUBLE + pipelinechannel--dnn_learning_rate: + parameterType: NUMBER_DOUBLE + pipelinechannel--dnn_optimizer_type: + parameterType: STRING + pipelinechannel--embed_categories: + parameterType: BOOLEAN + pipelinechannel--enable_profiler: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--eval_frequency_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--eval_steps: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_selection_algorithm: + parameterType: STRING + pipelinechannel--hidden_units: + parameterType: STRING + pipelinechannel--l1_regularization_strength: + parameterType: NUMBER_DOUBLE + pipelinechannel--l2_regularization_strength: + parameterType: NUMBER_DOUBLE + pipelinechannel--l2_shrinkage_regularization_strength: + parameterType: NUMBER_DOUBLE + pipelinechannel--learning_rate: + parameterType: NUMBER_DOUBLE + pipelinechannel--location: + parameterType: STRING + pipelinechannel--materialized_examples_format: + parameterType: STRING + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--max_steps: + parameterType: NUMBER_INTEGER + pipelinechannel--max_train_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--measurement_selection_type: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--optimization_metric: + parameterType: STRING + pipelinechannel--optimizer_type: + parameterType: STRING + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--run_feature_selection: + parameterType: BOOLEAN + pipelinechannel--seed: + parameterType: NUMBER_INTEGER + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--tf_auto_transform_features: + parameterType: STRUCT + pipelinechannel--tf_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--tf_transform_execution_engine: + parameterType: STRING + pipelinechannel--tf_transformations_path: + parameterType: STRING + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--use_wide: + parameterType: BOOLEAN + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + pipelinechannel--worker_pool_specs_override: + parameterType: LIST + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: An artifact of a model which to upload a new version to. Only + specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: "The unmanaged container model to be uploaded. The Model can\ + \ be passed from an upstream step or imported via a KFP `dsl.importer`.\n\ + :Examples:\n ::\n\n from kfp import dsl\n from google_cloud_pipeline_components.google_cloud_pipeline_components.types\ + \ import artifact_types\n\n importer_spec = dsl.importer(\n artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model',\n\ + \ artifact_class=artifact_types.UnmanagedContainerModel,\n metadata={\n\ + \ 'containerSpec': { 'imageUri':\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod'\n\ + \ }\n })" + isOptional: true + parameters: + description: + defaultValue: '' + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + isOptional: true + parameterType: STRING + display_name: + description: 'The display name of the Model. The name + + can be up to 128 characters long and can be consist of any UTF-8 + + characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model)' + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key spec for a Model. If set, this Model and all sub-resources of this + + Model will be secured by this key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + description: 'Metadata describing the Model''s + + input and output for explanation. Both `explanation_metadata` and `explanation_parameters` + must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata)' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters)' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your model. Label keys and values can be no longer than 64 + + characters (Unicode codepoints), can only contain lowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. See https://goo.gl/xmQnxf for more information and + + examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Optional location to upload this Model to. If + + not set, defaults to `us-central1`.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to upload this Model to. Defaults to the project in + which the PipelineJob is run. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: Artifact tracking the created Model. + parameters: + gcp_resources: + description: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) + which tracks the upload Model's long-running operation. + parameterType: STRING + comp-parse-worker-pool-specs-override: + executorLabel: exec-parse-worker-pool-specs-override + inputDefinitions: + parameters: + worker_pool_specs_override: + description: 'The list of dictionaries for overriding training + + and evaluation worker pool specs.' + parameterType: LIST + outputDefinitions: + parameters: + eval_machine_spec: + description: The eval machine spec. + parameterType: STRUCT + eval_replica_count: + description: The replica count for eval. + parameterType: NUMBER_INTEGER + training_disk_spec: + description: The training disk spec. + parameterType: STRUCT + training_machine_spec: + description: The training machine spec. + parameterType: STRUCT + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + comp-wide-and-deep-trainer: + executorLabel: exec-wide-and-deep-trainer + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to a JSON file for baseline values. + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized validation split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the materialized train split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Amount of time in seconds to run the trainer for. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to the training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The path to transform output. + parameters: + batch_size: + defaultValue: 100.0 + description: Batch size for training. + isOptional: true + parameterType: NUMBER_INTEGER + beta_1: + defaultValue: 0.9 + description: Beta 1 value for optimizer_type="adam". + isOptional: true + parameterType: NUMBER_DOUBLE + beta_2: + defaultValue: 0.999 + description: Beta 2 value for optimizer_type="adam". + isOptional: true + parameterType: NUMBER_DOUBLE + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to + + ''auto'', caching is determined based on the dataset size.' + isOptional: true + parameterType: STRING + dnn_beta_1: + defaultValue: 0.9 + description: Beta 1 value for dnn_optimizer_type="adam". + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_beta_2: + defaultValue: 0.999 + description: Beta 2 value for dnn_optimizer_type="adam". + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_dropout: + defaultValue: 0.0 + description: 'The probability we will drop out a given + + coordinate.' + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_l1_regularization_strength: + defaultValue: 0.0 + description: 'L1 regularization + + strength for dnn_optimizer_type="ftrl".' + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_l2_regularization_strength: + defaultValue: 0.0 + description: 'L2 regularization + + strength for dnn_optimizer_type="ftrl".' + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_l2_shrinkage_regularization_strength: + defaultValue: 0.0 + description: 'L2 shrinkage + + regularization strength for dnn_optimizer_type="ftrl".' + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_learning_rate: + description: 'The learning rate for training the + + deep part of the model.' + parameterType: NUMBER_DOUBLE + dnn_optimizer_type: + defaultValue: ftrl + description: 'The type of optimizer to use for the + + deep part of the model. Choices are "adam", "ftrl" and "sgd". for the + + Adam, FTRL, and Gradient Descent Optimizers, respectively.' + isOptional: true + parameterType: STRING + embed_categories: + defaultValue: true + description: 'If set to true, the categorical columns + + will be used embedded and used in the deep part of the model. Embedding + + size is the square root of the column cardinality.' + isOptional: true + parameterType: BOOLEAN + enable_profiler: + defaultValue: false + description: 'Enables profiling and saves a trace + + during evaluation.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and + + checkpointing will take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not + + specified or negative, it means run evaluation on the whole validation + + dataset. If set to 0, it means run evaluation for a fixed number of + + samples.' + isOptional: true + parameterType: NUMBER_INTEGER + hidden_units: + defaultValue: 30,30,30 + description: 'Hidden layer sizes to use for DNN feature + + columns, provided in comma-separated layers.' + isOptional: true + parameterType: STRING + l1_regularization_strength: + defaultValue: 0.0 + description: 'L1 regularization strength + + for optimizer_type="ftrl".' + isOptional: true + parameterType: NUMBER_DOUBLE + l2_regularization_strength: + defaultValue: 0.0 + description: 'L2 regularization strength + + for optimizer_type="ftrl"' + isOptional: true + parameterType: NUMBER_DOUBLE + l2_shrinkage_regularization_strength: + defaultValue: 0.0 + description: 'L2 shrinkage + + regularization strength for optimizer_type="ftrl".' + isOptional: true + parameterType: NUMBER_DOUBLE + learning_rate: + description: The learning rate used by the linear optimizer. + parameterType: NUMBER_DOUBLE + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + max_steps: + defaultValue: -1.0 + description: Number of steps to run the trainer for. + isOptional: true + parameterType: NUMBER_INTEGER + max_train_secs: + defaultValue: -1.0 + description: 'Amount of time in seconds to run the + + trainer for.' + isOptional: true + parameterType: NUMBER_INTEGER + measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: 'Which measurement to use + + if/when the service automatically selects the final measurement from + + previously reported intermediate measurements. One of "BEST_MEASUREMENT" + + or "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + optimization_metric: + defaultValue: '' + description: 'Optimization metric used for + + `measurement_selection_type`. Default is "rmse" for regression and "auc" + + for classification.' + isOptional: true + parameterType: STRING + optimizer_type: + defaultValue: adam + description: 'The type of optimizer to use. Choices are + + "adam", "ftrl" and "sgd" for the Adam, FTRL, and Gradient Descent + + Optimizers, respectively.' + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to + + produce. "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + target_column: + description: The target column name. + parameterType: STRING + training_disk_spec: + defaultValue: + boot_disk_size_gb: 100.0 + boot_disk_type: pd-ssd + description: The training disk spec. + isOptional: true + parameterType: STRUCT + training_machine_spec: + defaultValue: + machine_type: c2-standard-16 + description: 'The training machine + + spec. See https://cloud.google.com/compute/docs/machine-types for + + options.' + isOptional: true + parameterType: STRUCT + use_wide: + defaultValue: true + description: 'If set to true, the categorical columns will be + + used in the wide part of the DNN model.' + isOptional: true + parameterType: BOOLEAN + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: The UnmanagedContainerModel artifact. + parameters: + gcp_resources: + description: Serialized gcp_resources proto tracking the custom training + job. + parameterType: STRING +deploymentSpec: + executors: + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-infra-validator: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"pipeline_job\": + \"", "projects/{{$.inputs.parameters[''project'']}}/locations/{{$.inputs.parameters[''location'']}}/pipelineJobs/{{$.pipeline_job_uuid}}", + "\"", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.model.upload_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-parse-worker-pool-specs-override: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _parse_worker_pool_specs_override + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _parse_worker_pool_specs_override(\n worker_pool_specs_override:\ + \ list, # pylint:disable=g-bare-generic\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('training_machine_spec', dict), # pylint:disable=g-bare-generic\n\ + \ ('training_disk_spec', dict),\n ('eval_machine_spec', dict),\ + \ # pylint:disable=g-bare-generic\n ('eval_replica_count', int),\n\ + \ ],\n):\n \"\"\"Parses worker_pool_specs_override and returns training\ + \ and evaluation machine specifications.\n\n Args:\n worker_pool_specs_override:\ + \ The list of dictionaries for overriding training\n and evaluation\ + \ worker pool specs.\n\n Returns:\n training_machine_spec: The training\ + \ machine spec.\n training_disk_spec: The training disk spec.\n \ + \ eval_machine_spec: The eval machine spec.\n eval_replica_count:\ + \ The replica count for eval.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n training_machine_spec = {'machine_type': 'c2-standard-16'}\n training_disk_spec\ + \ = {'boot_disk_type': 'pd-ssd', 'boot_disk_size_gb': 100}\n eval_machine_spec\ + \ = {'machine_type': 'c2-standard-8'}\n eval_replica_count = 1\n\n if\ + \ worker_pool_specs_override:\n if len(worker_pool_specs_override) >=\ + \ 1 and isinstance(\n worker_pool_specs_override[0], dict\n ):\n\ + \ training_machine_spec = worker_pool_specs_override[0].get(\n \ + \ 'machine_spec', training_machine_spec\n )\n training_disk_spec\ + \ = worker_pool_specs_override[0].get(\n 'disk_spec', training_disk_spec\n\ + \ )\n if len(worker_pool_specs_override) == 4 and isinstance(\n\ + \ worker_pool_specs_override[3], dict\n ):\n eval_machine_spec\ + \ = worker_pool_specs_override[3].get(\n 'machine_spec', eval_machine_spec\n\ + \ )\n eval_replica_count = worker_pool_specs_override[3].get(\n\ + \ 'replica_count', eval_replica_count\n )\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'training_machine_spec',\n \ + \ 'training_disk_spec',\n 'eval_machine_spec',\n 'eval_replica_count',\n\ + \ ],\n )(\n training_machine_spec,\n training_disk_spec,\n\ + \ eval_machine_spec,\n eval_replica_count,\n )\n\n" + image: python:3.7 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + exec-wide-and-deep-trainer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"wide-and-deep-trainer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"", "1", + "\", \"machine_spec\": ", "{{$.inputs.parameters[''training_machine_spec'']}}", + ", \"disk_spec\": ", "{{$.inputs.parameters[''training_disk_spec'']}}", + ", \"container_spec\": {\"image_uri\":\"", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/wide-and-deep-training:20231002_0125", + "\", \"args\": [\"--target_column=", "{{$.inputs.parameters[''target_column'']}}", + "\", \"--weight_column=", "{{$.inputs.parameters[''weight_column'']}}", + "\", \"--model_type=", "{{$.inputs.parameters[''prediction_type'']}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_schema_path=", "{{$.inputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--job_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--training_data_path=", "{{$.inputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--validation_data_path=", "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--max_steps=", "{{$.inputs.parameters[''max_steps'']}}", "\", \"--max_train_secs=", + "{{$.inputs.parameters[''max_train_secs'']}}", "\", \"--learning_rate=", + "{{$.inputs.parameters[''learning_rate'']}}", "\", \"--optimizer_type=", + "{{$.inputs.parameters[''optimizer_type'']}}", "\", \"--l1_regularization_strength=", + "{{$.inputs.parameters[''l1_regularization_strength'']}}", "\", \"--l2_regularization_strength=", + "{{$.inputs.parameters[''l2_regularization_strength'']}}", "\", \"--l2_shrinkage_regularization_strength=", + "{{$.inputs.parameters[''l2_shrinkage_regularization_strength'']}}", "\", + \"--beta_1=", "{{$.inputs.parameters[''beta_1'']}}", "\", \"--beta_2=", + "{{$.inputs.parameters[''beta_2'']}}", "\", \"--hidden_units=", "{{$.inputs.parameters[''hidden_units'']}}", + "\", \"--use_wide=", "{{$.inputs.parameters[''use_wide'']}}", "\", \"--embed_categories=", + "{{$.inputs.parameters[''embed_categories'']}}", "\", \"--dnn_dropout=", + "{{$.inputs.parameters[''dnn_dropout'']}}", "\", \"--dnn_learning_rate=", + "{{$.inputs.parameters[''dnn_learning_rate'']}}", "\", \"--dnn_optimizer_type=", + "{{$.inputs.parameters[''dnn_optimizer_type'']}}", "\", \"--dnn_l1_regularization_strength=", + "{{$.inputs.parameters[''dnn_l1_regularization_strength'']}}", "\", \"--dnn_l2_regularization_strength=", + "{{$.inputs.parameters[''dnn_l2_regularization_strength'']}}", "\", \"--dnn_l2_shrinkage_regularization_strength=", + "{{$.inputs.parameters[''dnn_l2_shrinkage_regularization_strength'']}}", + "\", \"--dnn_beta_1=", "{{$.inputs.parameters[''dnn_beta_1'']}}", "\", \"--dnn_beta_2=", + "{{$.inputs.parameters[''dnn_beta_2'']}}", "\", \"--enable_profiler=", "{{$.inputs.parameters[''enable_profiler'']}}", + "\", \"--cache_data=", "{{$.inputs.parameters[''cache_data'']}}", "\", \"--seed=", + "{{$.inputs.parameters[''seed'']}}", "\", \"--eval_steps=", "{{$.inputs.parameters[''eval_steps'']}}", + "\", \"--batch_size=", "{{$.inputs.parameters[''batch_size'']}}", "\", \"--measurement_selection_type=", + "{{$.inputs.parameters[''measurement_selection_type'']}}", "\", \"--optimization_metric=", + "{{$.inputs.parameters[''optimization_metric'']}}", "\", \"--eval_frequency_secs=", + "{{$.inputs.parameters[''eval_frequency_secs'']}}", "\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 +pipelineInfo: + description: 'Train a model using the Tabular Workflow for Wide & Deep pipelines. + + Wide & Deep jointly trains wide linear models and deep neural networks. It + + combines the benefits of memorization and generalization.' + name: automl-tabular-wide-and-deep-trainer +root: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--batch_size: + componentInputParameter: batch_size + pipelinechannel--beta_1: + componentInputParameter: beta_1 + pipelinechannel--beta_2: + componentInputParameter: beta_2 + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + pipelinechannel--cache_data: + componentInputParameter: cache_data + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + pipelinechannel--dataset_level_transformations: + componentInputParameter: dataset_level_transformations + pipelinechannel--dnn_beta_1: + componentInputParameter: dnn_beta_1 + pipelinechannel--dnn_beta_2: + componentInputParameter: dnn_beta_2 + pipelinechannel--dnn_dropout: + componentInputParameter: dnn_dropout + pipelinechannel--dnn_l1_regularization_strength: + componentInputParameter: dnn_l1_regularization_strength + pipelinechannel--dnn_l2_regularization_strength: + componentInputParameter: dnn_l2_regularization_strength + pipelinechannel--dnn_l2_shrinkage_regularization_strength: + componentInputParameter: dnn_l2_shrinkage_regularization_strength + pipelinechannel--dnn_learning_rate: + componentInputParameter: dnn_learning_rate + pipelinechannel--dnn_optimizer_type: + componentInputParameter: dnn_optimizer_type + pipelinechannel--embed_categories: + componentInputParameter: embed_categories + pipelinechannel--enable_profiler: + componentInputParameter: enable_profiler + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--eval_frequency_secs: + componentInputParameter: eval_frequency_secs + pipelinechannel--eval_steps: + componentInputParameter: eval_steps + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + pipelinechannel--hidden_units: + componentInputParameter: hidden_units + pipelinechannel--l1_regularization_strength: + componentInputParameter: l1_regularization_strength + pipelinechannel--l2_regularization_strength: + componentInputParameter: l2_regularization_strength + pipelinechannel--l2_shrinkage_regularization_strength: + componentInputParameter: l2_shrinkage_regularization_strength + pipelinechannel--learning_rate: + componentInputParameter: learning_rate + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--materialized_examples_format: + componentInputParameter: materialized_examples_format + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--max_steps: + componentInputParameter: max_steps + pipelinechannel--max_train_secs: + componentInputParameter: max_train_secs + pipelinechannel--measurement_selection_type: + componentInputParameter: measurement_selection_type + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--optimization_metric: + componentInputParameter: optimization_metric + pipelinechannel--optimizer_type: + componentInputParameter: optimizer_type + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--prediction_type: + componentInputParameter: prediction_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--run_feature_selection: + componentInputParameter: run_feature_selection + pipelinechannel--seed: + componentInputParameter: seed + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + pipelinechannel--tf_custom_transformation_definitions: + componentInputParameter: tf_custom_transformation_definitions + pipelinechannel--tf_transform_execution_engine: + componentInputParameter: tf_transform_execution_engine + pipelinechannel--tf_transformations_path: + componentInputParameter: tf_transformations_path + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--use_wide: + componentInputParameter: use_wide + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + pipelinechannel--worker_pool_specs_override: + componentInputParameter: worker_pool_specs_override + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Parent model if this model is uploaded as a version. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + batch_size: + defaultValue: 100.0 + description: Batch size for training. + isOptional: true + parameterType: NUMBER_INTEGER + beta_1: + defaultValue: 0.9 + description: Beta 1 value for optimizer_type='adam'. + isOptional: true + parameterType: NUMBER_DOUBLE + beta_2: + defaultValue: 0.999 + description: Beta 2 value for optimizer_type='adam'. + isOptional: true + parameterType: NUMBER_DOUBLE + bigquery_staging_full_dataset_id: + defaultValue: '' + description: Staging directory for BigQuery tables. + isOptional: true + parameterType: STRING + cache_data: + defaultValue: auto + description: 'Whether to cache data or not. If set to ''auto'', caching is + + determined based on the dataset size.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + description: 'Dataset-level custom + + transformation definitions in string format.' + isOptional: true + parameterType: LIST + dataset_level_transformations: + description: 'Dataset-level transformation configuration in + + string format.' + isOptional: true + parameterType: LIST + dnn_beta_1: + defaultValue: 0.9 + description: Beta 1 value for dnn_optimizer_type='adam'. + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_beta_2: + defaultValue: 0.999 + description: Beta 2 value for dnn_optimizer_type='adam'. + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_dropout: + defaultValue: 0.0 + description: The probability we will drop out a given coordinate. + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_l1_regularization_strength: + defaultValue: 0.0 + description: 'L1 regularization strength for + + dnn_optimizer_type=''ftrl''.' + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_l2_regularization_strength: + defaultValue: 0.0 + description: 'L2 regularization strength for + + dnn_optimizer_type=''ftrl''.' + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_l2_shrinkage_regularization_strength: + defaultValue: 0.0 + description: 'L2 shrinkage regularization + + strength for dnn_optimizer_type=''ftrl''.' + isOptional: true + parameterType: NUMBER_DOUBLE + dnn_learning_rate: + description: 'The learning rate for training the deep part of the + + model.' + parameterType: NUMBER_DOUBLE + dnn_optimizer_type: + defaultValue: adam + description: 'The type of optimizer to use for the deep part of the + + model. Choices are ''adam'', ''ftrl'' and ''sgd''. for the Adam, FTRL, and + + Gradient Descent Optimizers, respectively.' + isOptional: true + parameterType: STRING + embed_categories: + defaultValue: true + description: 'If set to true, the categorical columns will be used + + embedded and used in the deep part of the model. Embedding size is the + + square root of the column cardinality.' + isOptional: true + parameterType: BOOLEAN + enable_profiler: + defaultValue: false + description: Enables profiling and saves a trace during evaluation. + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_frequency_secs: + defaultValue: 600.0 + description: 'Frequency at which evaluation and checkpointing will + + take place.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_steps: + defaultValue: 0.0 + description: 'Number of steps to run evaluation for. If not specified or + + negative, it means run evaluation on the whole validation dataset. If set + + to 0, it means run evaluation for a fixed number of samples.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_selection_algorithm: + defaultValue: AMI + description: Feature selection algorithm. + isOptional: true + parameterType: STRING + hidden_units: + defaultValue: 30,30,30 + description: 'Hidden layer sizes to use for DNN feature columns, provided + in + + comma-separated layers.' + isOptional: true + parameterType: STRING + l1_regularization_strength: + defaultValue: 0.0 + description: 'L1 regularization strength for + + optimizer_type=''ftrl''.' + isOptional: true + parameterType: NUMBER_DOUBLE + l2_regularization_strength: + defaultValue: 0.0 + description: 'L2 regularization strength for + + optimizer_type=''ftrl''.' + isOptional: true + parameterType: NUMBER_DOUBLE + l2_shrinkage_regularization_strength: + defaultValue: 0.0 + description: 'L2 shrinkage regularization strength + + for optimizer_type=''ftrl''.' + isOptional: true + parameterType: NUMBER_DOUBLE + learning_rate: + description: The learning rate used by the linear optimizer. + parameterType: NUMBER_DOUBLE + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: The format for the materialized examples. + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: -1.0 + description: Maximum number of features to select. + isOptional: true + parameterType: NUMBER_INTEGER + max_steps: + defaultValue: -1.0 + description: Number of steps to run the trainer for. + isOptional: true + parameterType: NUMBER_INTEGER + max_train_secs: + defaultValue: -1.0 + description: Amount of time in seconds to run the trainer for. + isOptional: true + parameterType: NUMBER_INTEGER + measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: 'Which measurement to use if/when the service + + automatically selects the final measurement from previously reported + + intermediate measurements. One of "BEST_MEASUREMENT" or + + "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + optimization_metric: + defaultValue: '' + description: 'Optimization metric used for + + `measurement_selection_type`. Default is "rmse" for regression and "auc" + + for classification.' + isOptional: true + parameterType: STRING + optimizer_type: + defaultValue: adam + description: 'The type of optimizer to use. Choices are "adam", "ftrl" and + + "sgd" for the Adam, FTRL, and Gradient Descent Optimizers, respectively.' + isOptional: true + parameterType: STRING + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to produce. + + "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: Whether to enable feature selection. + isOptional: true + parameterType: BOOLEAN + seed: + defaultValue: 1.0 + description: Seed to be used for this run. + isOptional: true + parameterType: NUMBER_INTEGER + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: Test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + description: List of auto transform features. + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + description: 'TF custom transformation definitions + + in string format.' + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: '' + description: 'Execution engine to run TF-based + + transformations. Currently supports "dataflow" or "bigquery"' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: Path to TF transformation configuration. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + use_wide: + defaultValue: true + description: 'If set to true, the categorical columns will be used in the + wide + + part of the DNN model.' + isOptional: true + parameterType: BOOLEAN + validation_fraction: + defaultValue: -1.0 + description: Validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + worker_pool_specs_override: + description: 'The dictionary for overriding training and + + evaluation worker pool specs. The dictionary should be of format + + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job.py new file mode 100644 index 0000000000..b5f3ed2357 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job.py @@ -0,0 +1,108 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML XGBoost Hyperparameter Tuning component spec.""" + +from typing import Optional + +from kfp import dsl + + +@dsl.container_component +def xgboost_hyperparameter_tuning_job( + project: str, + location: str, + study_spec_metric_id: str, + study_spec_metric_goal: str, + study_spec_parameters_override: list, + max_trial_count: int, + parallel_trial_count: int, + worker_pool_specs: list, + gcp_resources: dsl.OutputPath(str), + max_failed_trial_count: Optional[int] = 0, + study_spec_algorithm: Optional[str] = 'ALGORITHM_UNSPECIFIED', + study_spec_measurement_selection_type: Optional[str] = 'BEST_MEASUREMENT', + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Tunes XGBoost hyperparameters using Vertex HyperparameterTuningJob API. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + study_spec_metric_id: Metric to optimize. For options, please look under 'eval_metric' at https://xgboost.readthedocs.io/en/stable/parameter.html#learning-task-parameters. + study_spec_metric_goal: Optimization goal of the metric, possible values: "MAXIMIZE", "MINIMIZE". + study_spec_parameters_override: List of dictionaries representing parameters to optimize. The dictionary key is the parameter_id, which is passed to training job as a command line argument, and the dictionary value is the parameter specification of the metric. + max_trial_count: The desired total number of trials. + parallel_trial_count: The desired number of trials to run in parallel. + max_failed_trial_count: The number of failed trials that need to be seen before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides how many trials must fail before the whole job fails. + study_spec_algorithm: The search algorithm specified for the study. One of 'ALGORITHM_UNSPECIFIED', 'GRID_SEARCH', or 'RANDOM_SEARCH'. + study_spec_measurement_selection_type: Which measurement to use if/when the service automatically selects the final measurement from previously reported intermediate measurements. One of "BEST_MEASUREMENT" or "LAST_MEASUREMENT". + worker_pool_specs: The worker pool specs. + encryption_spec_key_name: The KMS key name. + + Returns: + gcp_resources: Serialized gcp_resources proto tracking the custom training job. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.hyperparameter_tuning_job.launcher', + ], + args=[ + '--type', + 'HyperparameterTuningJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "xgboost-hyperparameter-tuning-job-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + '"}, "study_spec": {"metrics": [{"metric_id": "', + study_spec_metric_id, + '", "goal": "', + study_spec_metric_goal, + '"}], "parameters": ', + study_spec_parameters_override, + ', "algorithm": "', + study_spec_algorithm, + '", "measurement_selection_type": "', + study_spec_measurement_selection_type, + '"}, "max_trial_count": ', + max_trial_count, + ', "parallel_trial_count": ', + parallel_trial_count, + ', "max_failed_trial_count": ', + max_failed_trial_count, + ', "trial_job_spec": {"worker_pool_specs": ', + worker_pool_specs, + '}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml new file mode 100644 index 0000000000..79c07e0e19 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_hyperparameter_tuning_job_pipeline.yaml @@ -0,0 +1,4406 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-xgboost-hyperparameter-tuning-job +# Description: The XGBoost HyperparameterTuningJob pipeline. +# Inputs: +# bigquery_staging_full_dataset_id: str [Default: ''] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# disable_default_eval_metric: int [Default: 0.0] +# encryption_spec_key_name: str [Default: ''] +# eval_metric: str [Default: ''] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# feature_selection_algorithm: str [Default: 'AMI'] +# location: str +# max_failed_trial_count: int [Default: 0.0] +# max_selected_features: int [Default: -1.0] +# max_trial_count: int +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# objective: str +# parallel_trial_count: int +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# project: str +# root_dir: str +# run_evaluation: bool [Default: False] +# run_feature_selection: bool [Default: False] +# seed: int [Default: 0.0] +# seed_per_iteration: bool [Default: False] +# stratified_split_key: str [Default: ''] +# study_spec_algorithm: str [Default: 'ALGORITHM_UNSPECIFIED'] +# study_spec_measurement_selection_type: str [Default: 'BEST_MEASUREMENT'] +# study_spec_metric_goal: str +# study_spec_metric_id: str +# study_spec_parameters_override: list +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# tf_custom_transformation_definitions: list +# tf_transformations_path: str [Default: ''] +# training_accelerator_count: int [Default: 0.0] +# training_accelerator_type: str [Default: ''] +# training_fraction: float [Default: -1.0] +# training_machine_type: str [Default: 'c2-standard-16'] +# training_total_replica_count: int [Default: 1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# Outputs: +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--get-prediction-type-for-xgboost-Output + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + inputDefinitions: + artifacts: + pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--get-prediction-type-for-xgboost-Output: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - bool-identity + - feature-transform-engine + - get-best-hyperparameter-tuning-job-trial + - get-prediction-type-for-xgboost + inputs: + artifacts: + pipelinechannel--get-best-hyperparameter-tuning-job-trial-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + parameters: + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--get-prediction-type-for-xgboost-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-type-for-xgboost + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + dependentTasks: + - get-prediction-type-for-xgboost + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: pipelinechannel--dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: pipelinechannel--dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: pipelinechannel--feature_selection_algorithm + location: + componentInputParameter: pipelinechannel--location + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-type-for-xgboost + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_feature_selection: + componentInputParameter: pipelinechannel--run_feature_selection + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--tf_auto_transform_features + tf_custom_transformation_definitions: + componentInputParameter: pipelinechannel--tf_custom_transformation_definitions + tf_transformations_path: + componentInputParameter: pipelinechannel--tf_transformations_path + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + generate-xgboost-hyperparameter-tuning-worker-pool-specs: + cachingOptions: + enableCache: true + componentRef: + name: comp-generate-xgboost-hyperparameter-tuning-worker-pool-specs + dependentTasks: + - feature-transform-engine + - split-materialized-data + - training-configurator-and-validator + inputs: + artifacts: + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + training_schema_uri: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + parameters: + accelerator_count: + componentInputParameter: pipelinechannel--training_accelerator_count + accelerator_type: + componentInputParameter: pipelinechannel--training_accelerator_type + disable_default_eval_metric: + componentInputParameter: pipelinechannel--disable_default_eval_metric + eval_metric: + componentInputParameter: pipelinechannel--eval_metric + machine_type: + componentInputParameter: pipelinechannel--training_machine_type + objective: + componentInputParameter: pipelinechannel--objective + seed: + componentInputParameter: pipelinechannel--seed + seed_per_iteration: + componentInputParameter: pipelinechannel--seed_per_iteration + target_column: + componentInputParameter: pipelinechannel--target_column + total_replica_count: + componentInputParameter: pipelinechannel--training_total_replica_count + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: generate-xgboost-hyperparameter-tuning-worker-pool-specs + get-best-hyperparameter-tuning-job-trial: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-best-hyperparameter-tuning-job-trial + dependentTasks: + - generate-xgboost-hyperparameter-tuning-worker-pool-specs + - xgboost-hyperparameter-tuning-job + inputs: + parameters: + gcp_resources: + taskOutputParameter: + outputParameterKey: gcp_resources + producerTask: xgboost-hyperparameter-tuning-job + instance_schema_uri: + taskOutputParameter: + outputParameterKey: instance_schema_path + producerTask: generate-xgboost-hyperparameter-tuning-worker-pool-specs + prediction_docker_uri: + taskOutputParameter: + outputParameterKey: prediction_docker_uri_artifact_path + producerTask: generate-xgboost-hyperparameter-tuning-worker-pool-specs + prediction_schema_uri: + taskOutputParameter: + outputParameterKey: prediction_schema_path + producerTask: generate-xgboost-hyperparameter-tuning-worker-pool-specs + read_value_from_file: + runtimeValue: + constant: true + study_spec_metric_goal: + componentInputParameter: pipelinechannel--study_spec_metric_goal + trials_dir: + taskOutputParameter: + outputParameterKey: trials_path + producerTask: generate-xgboost-hyperparameter-tuning-worker-pool-specs + taskInfo: + name: get-best-hyperparameter-tuning-job-trial + get-prediction-type-for-xgboost: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-type-for-xgboost + inputs: + parameters: + objective: + componentInputParameter: pipelinechannel--objective + taskInfo: + name: get-prediction-type-for-xgboost + get-xgboost-study-spec-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-xgboost-study-spec-parameters + inputs: + parameters: + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + taskInfo: + name: get-xgboost-study-spec-parameters + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - get-best-hyperparameter-tuning-job-trial + inputs: + artifacts: + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: get-best-hyperparameter-tuning-job-trial + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + - get-prediction-type-for-xgboost + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + prediction_type: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-type-for-xgboost + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + xgboost-hyperparameter-tuning-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-xgboost-hyperparameter-tuning-job + dependentTasks: + - generate-xgboost-hyperparameter-tuning-worker-pool-specs + - get-xgboost-study-spec-parameters + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + max_failed_trial_count: + componentInputParameter: pipelinechannel--max_failed_trial_count + max_trial_count: + componentInputParameter: pipelinechannel--max_trial_count + parallel_trial_count: + componentInputParameter: pipelinechannel--parallel_trial_count + project: + componentInputParameter: pipelinechannel--project + study_spec_algorithm: + componentInputParameter: pipelinechannel--study_spec_algorithm + study_spec_measurement_selection_type: + componentInputParameter: pipelinechannel--study_spec_measurement_selection_type + study_spec_metric_goal: + componentInputParameter: pipelinechannel--study_spec_metric_goal + study_spec_metric_id: + componentInputParameter: pipelinechannel--study_spec_metric_id + study_spec_parameters_override: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-xgboost-study-spec-parameters + worker_pool_specs: + taskOutputParameter: + outputParameterKey: worker_pool_specs + producerTask: generate-xgboost-hyperparameter-tuning-worker-pool-specs + taskInfo: + name: xgboost-hyperparameter-tuning-job + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--dataset_level_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--dataset_level_transformations: + parameterType: LIST + pipelinechannel--disable_default_eval_metric: + parameterType: NUMBER_INTEGER + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--eval_metric: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_selection_algorithm: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--max_failed_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--max_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--objective: + parameterType: STRING + pipelinechannel--parallel_trial_count: + parameterType: NUMBER_INTEGER + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--run_feature_selection: + parameterType: BOOLEAN + pipelinechannel--seed: + parameterType: NUMBER_INTEGER + pipelinechannel--seed_per_iteration: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--study_spec_algorithm: + parameterType: STRING + pipelinechannel--study_spec_measurement_selection_type: + parameterType: STRING + pipelinechannel--study_spec_metric_goal: + parameterType: STRING + pipelinechannel--study_spec_metric_id: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--tf_auto_transform_features: + parameterType: STRUCT + pipelinechannel--tf_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--tf_transformations_path: + parameterType: STRING + pipelinechannel--training_accelerator_count: + parameterType: NUMBER_INTEGER + pipelinechannel--training_accelerator_type: + parameterType: STRING + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_machine_type: + parameterType: STRING + pipelinechannel--training_total_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-generate-xgboost-hyperparameter-tuning-worker-pool-specs: + executorLabel: exec-generate-xgboost-hyperparameter-tuning-worker-pool-specs + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path to JSON file for baseline values. + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Required. The path to the materialized validation + + split.' + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Required. The path to the materialized train + + split.' + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Required. The path to the training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Required. The path to transform output. + parameters: + accelerator_count: + defaultValue: 0.0 + description: Accelerator count. + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: Accelerator type. + isOptional: true + parameterType: STRING + disable_default_eval_metric: + defaultValue: 0.0 + description: 'Flag to disable default metric. Set to >0 to + + disable. Default to 0.' + isOptional: true + parameterType: NUMBER_INTEGER + eval_metric: + defaultValue: '' + description: 'Evaluation metrics for validation data represented as a + + comma-separated string.' + isOptional: true + parameterType: STRING + machine_type: + defaultValue: c2-standard-16 + description: Machine type. + isOptional: true + parameterType: STRING + objective: + description: Required. Specifies the learning task and the learning objective. + parameterType: STRING + seed: + defaultValue: 0.0 + description: Random seed. + isOptional: true + parameterType: NUMBER_INTEGER + seed_per_iteration: + defaultValue: false + description: Seed PRNG determnisticly via iterator number. + isOptional: true + parameterType: BOOLEAN + target_column: + description: Required. Target column name. + parameterType: STRING + total_replica_count: + description: Number of workers. + parameterType: NUMBER_INTEGER + weight_column: + defaultValue: '' + description: Weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + job_dir: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + instance_schema_path: + parameterType: STRING + instance_schema_uri: + parameterType: STRING + prediction_docker_uri_artifact_path: + parameterType: STRING + prediction_docker_uri_output: + parameterType: STRING + prediction_schema_path: + parameterType: STRING + prediction_schema_uri: + parameterType: STRING + trials: + parameterType: STRING + trials_path: + parameterType: STRING + worker_pool_specs: + parameterType: LIST + comp-get-best-hyperparameter-tuning-job-trial: + executorLabel: exec-get-best-hyperparameter-tuning-job-trial + inputDefinitions: + parameters: + gcp_resources: + description: Proto tracking the hyperparameter tuning job. + parameterType: STRING + instance_schema_uri: + defaultValue: '' + description: The instance schema uri. + isOptional: true + parameterType: STRING + prediction_docker_uri: + defaultValue: '' + description: The prediction docker container uri. + isOptional: true + parameterType: STRING + prediction_schema_uri: + defaultValue: '' + description: The prediction schema_uri. + isOptional: true + parameterType: STRING + read_value_from_file: + defaultValue: false + description: If true, read file to get the relevant value. + isOptional: true + parameterType: BOOLEAN + study_spec_metric_goal: + description: 'Optimization goal of the metric, possible values: + + "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + trials_dir: + defaultValue: '' + description: The path to the hyperparameter tuning trials. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-get-prediction-type-for-xgboost: + executorLabel: exec-get-prediction-type-for-xgboost + inputDefinitions: + parameters: + objective: + description: The XGBoost training objective + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-xgboost-study-spec-parameters: + executorLabel: exec-get-xgboost-study-spec-parameters + inputDefinitions: + parameters: + study_spec_parameters_override: + description: 'List of dictionaries representing parameters + + to optimize. The dictionary key is the parameter_id, which is passed to + + training job as a command line argument, and the dictionary value is the + + parameter specification of the metric.' + parameterType: LIST + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: An artifact of a model which to upload a new version to. Only + specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: "The unmanaged container model to be uploaded. The Model can\ + \ be passed from an upstream step or imported via a KFP `dsl.importer`.\n\ + :Examples:\n ::\n\n from kfp import dsl\n from google_cloud_pipeline_components.google_cloud_pipeline_components.types\ + \ import artifact_types\n\n importer_spec = dsl.importer(\n artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model',\n\ + \ artifact_class=artifact_types.UnmanagedContainerModel,\n metadata={\n\ + \ 'containerSpec': { 'imageUri':\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod'\n\ + \ }\n })" + isOptional: true + parameters: + description: + defaultValue: '' + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + isOptional: true + parameterType: STRING + display_name: + description: 'The display name of the Model. The name + + can be up to 128 characters long and can be consist of any UTF-8 + + characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model)' + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key spec for a Model. If set, this Model and all sub-resources of this + + Model will be secured by this key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + description: 'Metadata describing the Model''s + + input and output for explanation. Both `explanation_metadata` and `explanation_parameters` + must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata)' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters)' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your model. Label keys and values can be no longer than 64 + + characters (Unicode codepoints), can only contain lowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. See https://goo.gl/xmQnxf for more information and + + examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Optional location to upload this Model to. If + + not set, defaults to `us-central1`.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to upload this Model to. Defaults to the project in + which the PipelineJob is run. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: Artifact tracking the created Model. + parameters: + gcp_resources: + description: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) + which tracks the upload Model's long-running operation. + parameterType: STRING + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + comp-xgboost-hyperparameter-tuning-job: + executorLabel: exec-xgboost-hyperparameter-tuning-job + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + max_failed_trial_count: + defaultValue: 0.0 + description: 'The number of failed trials that + + need to be seen before failing the HyperparameterTuningJob. If set to + 0, + + Vertex AI decides how many trials must fail before the whole job fails.' + isOptional: true + parameterType: NUMBER_INTEGER + max_trial_count: + description: The desired total number of trials. + parameterType: NUMBER_INTEGER + parallel_trial_count: + description: 'The desired number of trials to run + + in parallel.' + parameterType: NUMBER_INTEGER + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + study_spec_algorithm: + defaultValue: ALGORITHM_UNSPECIFIED + description: 'The search algorithm specified for + + the study. One of ''ALGORITHM_UNSPECIFIED'', ''GRID_SEARCH'', or + + ''RANDOM_SEARCH''.' + isOptional: true + parameterType: STRING + study_spec_measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: 'Which measurement + + to use if/when the service automatically selects the final measurement + + from previously reported intermediate measurements. One of + + "BEST_MEASUREMENT" or "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + study_spec_metric_goal: + description: 'Optimization goal of the metric, + + possible values: "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + study_spec_metric_id: + description: 'Metric to optimize. For options, + + please look under ''eval_metric'' at + + https://xgboost.readthedocs.io/en/stable/parameter.html#learning-task-parameters.' + parameterType: STRING + study_spec_parameters_override: + description: 'List of dictionaries + + representing parameters to optimize. The dictionary key is the + + parameter_id, which is passed to training job as a command line + + argument, and the dictionary value is the parameter specification of the + + metric.' + parameterType: LIST + worker_pool_specs: + description: The worker pool specs. + parameterType: LIST + outputDefinitions: + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the custom training + + job.' + parameterType: STRING +deploymentSpec: + executors: + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-generate-xgboost-hyperparameter-tuning-worker-pool-specs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _generate_xgboost_hyperparameter_tuning_worker_pool_specs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _generate_xgboost_hyperparameter_tuning_worker_pool_specs(\n\ + \ total_replica_count: int,\n target_column: str,\n objective:\ + \ str,\n materialized_train_split: dsl.InputPath('MaterializedSplit'),\n\ + \ materialized_eval_split: dsl.InputPath('MaterializedSplit'),\n transform_output:\ + \ dsl.InputPath('TransformOutput'),\n training_schema_uri: dsl.InputPath('DatasetSchema'),\n\ + \ instance_baseline: dsl.InputPath('AutoMLTabularInstanceBaseline'),\n\ + \ job_dir: dsl.OutputPath('JobDir'),\n instance_schema_uri: dsl.OutputPath(str),\n\ + \ prediction_schema_uri: dsl.OutputPath(str),\n trials: dsl.OutputPath(str),\n\ + \ prediction_docker_uri_output: dsl.OutputPath(str),\n machine_type:\ + \ str = 'c2-standard-16',\n accelerator_type: str = '',\n accelerator_count:\ + \ int = 0,\n weight_column: str = '',\n eval_metric: str = '',\n \ + \ disable_default_eval_metric: int = 0,\n seed: int = 0,\n seed_per_iteration:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('worker_pool_specs',\ + \ list), # pylint:disable=g-bare-generic\n ('instance_schema_path',\ + \ str),\n ('prediction_schema_path', str),\n ('trials_path',\ + \ str),\n ('prediction_docker_uri_artifact_path', str),\n ],\n\ + ):\n \"\"\"Generates worker pool specs for XGBoost hyperparameter tuning.\n\ + \n For single machine XGBoost training, returns one worker pool spec for\ + \ master.\n For distributed XGBoost training, returns two worker pool specs,\ + \ the first one\n for master and the second one for the remaining workers.\n\ + \n Args:\n total_replica_count: Number of workers.\n target_column:\ + \ Required. Target column name.\n objective: Required. Specifies the\ + \ learning task and the learning objective.\n materialized_train_split:\ + \ Required. The path to the materialized train\n split.\n materialized_eval_split:\ + \ Required. The path to the materialized validation\n split.\n transform_output:\ + \ Required. The path to transform output.\n training_schema_uri: Required.\ + \ The path to the training schema.\n instance_baseline: Path to JSON\ + \ file for baseline values.\n job_dir: Job dir path.\n instance_schema_uri:\ + \ The instance schema uri.\n prediction_schema_uri: The prediction schema_uri.\n\ + \ trials: The trials uri.\n prediction_docker_uri_output: The prediction\ + \ docker container uri.\n machine_type: Machine type.\n accelerator_type:\ + \ Accelerator type.\n accelerator_count: Accelerator count.\n weight_column:\ + \ Weight column name.\n eval_metric: Evaluation metrics for validation\ + \ data represented as a\n comma-separated string.\n disable_default_eval_metric:\ + \ Flag to disable default metric. Set to >0 to\n disable. Default to\ + \ 0.\n seed: Random seed.\n seed_per_iteration: Seed PRNG determnisticly\ + \ via iterator number.\n\n Raises:\n ValueError: If accelerator_count\ + \ <= 0 and accelerator_type is specified.\n\n Returns:\n Output parameters.\n\ + \ \"\"\"\n import copy\n import collections\n import re\n\n def get_gcs_path(path):\n\ + \ return re.sub(r'^/gcs/', r'gs://', path)\n\n master_worker_pool_spec\ + \ = {\n 'replica_count': 1,\n 'machine_spec': {\n 'machine_type':\ + \ machine_type,\n },\n 'container_spec': {\n 'image_uri':\ + \ 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/xgboost-training:20231002_0125',\n\ + \ 'args': [\n f'--job_dir={get_gcs_path(job_dir)}',\n\ + \ f'--instance_schema_path={get_gcs_path(instance_schema_uri)}',\n\ + \ f'--prediction_schema_path={get_gcs_path(prediction_schema_uri)}',\n\ + \ f'--trials_path={get_gcs_path(trials)}',\n f'--prediction_docker_uri_artifact_path={get_gcs_path(prediction_docker_uri_output)}',\n\ + \ f'--target_column={target_column}',\n f'--objective={objective}',\n\ + \ f'--training_data_path={get_gcs_path(materialized_train_split)}',\n\ + \ f'--validation_data_path={get_gcs_path(materialized_eval_split)}',\n\ + \ f'--transform_output_path={get_gcs_path(transform_output)}',\n\ + \ f'--training_schema_path={get_gcs_path(training_schema_uri)}',\n\ + \ f'--baseline_path={get_gcs_path(instance_baseline)}',\n \ + \ f'--eval_metric={eval_metric}',\n f'--disable_default_eval_metric={disable_default_eval_metric}',\n\ + \ f'--seed={seed}',\n f'--seed_per_iteration={seed_per_iteration}',\n\ + \ '--prediction_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/xgboost-prediction-server:20231002_0125',\n\ + \ ],\n },\n }\n\n # Add optional arguments if set\n if\ + \ weight_column:\n master_worker_pool_spec['container_spec']['args'].append(\n\ + \ f'--weight_column={weight_column}'\n )\n\n # Add accelerator_type\ + \ and accelerator_count if set.\n if accelerator_type:\n if accelerator_count\ + \ <= 0:\n raise ValueError(\n 'Accelerator count must be greator\ + \ than 0 when type is specified.'\n )\n master_worker_pool_spec['machine_spec'][\n\ + \ 'accelerator_type'\n ] = accelerator_type\n master_worker_pool_spec['machine_spec'][\n\ + \ 'accelerator_count'\n ] = accelerator_count\n\n worker_pool_specs_lst\ + \ = [master_worker_pool_spec]\n\n # Add an additional worker pool spec\ + \ for distributed training.\n if total_replica_count > 1:\n additional_replica\ + \ = total_replica_count - 1\n additional_worker_spec = copy.deepcopy(master_worker_pool_spec)\n\ + \ additional_worker_spec['replica_count'] = additional_replica\n worker_pool_specs_lst.append(additional_worker_spec)\n\ + \n return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'worker_pool_specs',\n 'instance_schema_path',\n 'prediction_schema_path',\n\ + \ 'trials_path',\n 'prediction_docker_uri_artifact_path',\n\ + \ ],\n )(\n worker_pool_specs_lst,\n get_gcs_path(instance_schema_uri),\n\ + \ get_gcs_path(prediction_schema_uri),\n get_gcs_path(trials),\n\ + \ get_gcs_path(prediction_docker_uri_output),\n )\n\n" + image: python:3.7 + exec-get-best-hyperparameter-tuning-job-trial: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_best_hyperparameter_tuning_job_trial + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_best_hyperparameter_tuning_job_trial(\n gcp_resources:\ + \ str,\n study_spec_metric_goal: str,\n unmanaged_container_model:\ + \ dsl.Output[dsl.Artifact],\n trials_dir: str = '',\n instance_schema_uri:\ + \ str = '',\n prediction_schema_uri: str = '',\n prediction_docker_uri:\ + \ str = '',\n read_value_from_file: bool = False,\n):\n \"\"\"Gets best\ + \ HyperparameterTuningJob trial.\n\n Args:\n gcp_resources: Proto tracking\ + \ the hyperparameter tuning job.\n study_spec_metric_goal: Optimization\ + \ goal of the metric, possible values:\n \"MAXIMIZE\", \"MINIMIZE\"\ + .\n unmanaged_container_model: The unmanaged model.\n trials_dir:\ + \ The path to the hyperparameter tuning trials.\n instance_schema_uri:\ + \ The instance schema uri.\n prediction_schema_uri: The prediction schema_uri.\n\ + \ prediction_docker_uri: The prediction docker container uri.\n read_value_from_file:\ + \ If true, read file to get the relevant value.\n\n Raises:\n RuntimeError:\ + \ If there are multiple metrics.\n \"\"\"\n\n import os\n import json\n\ + \ from google.api_core.retry import Retry\n from google.cloud import aiplatform_v1beta1\ + \ as aip\n import tensorflow as tf\n\n # If path to file with value is\ + \ provided, read the file before continuing.\n if read_value_from_file:\n\ + \ with tf.io.gfile.GFile(trials_dir, 'r') as f:\n trials_dir = f.read()\n\ + \ with tf.io.gfile.GFile(instance_schema_uri, 'r') as f:\n instance_schema_uri\ + \ = f.read()\n with tf.io.gfile.GFile(prediction_schema_uri, 'r') as\ + \ f:\n prediction_schema_uri = f.read()\n with tf.io.gfile.GFile(prediction_docker_uri,\ + \ 'r') as f:\n prediction_docker_uri = f.read()\n\n api_endpoint_suffix\ + \ = '-aiplatform.googleapis.com'\n gcp_resources_json = json.loads(gcp_resources)\n\ + \ resource = gcp_resources_json['resources'][0]\n\n uri_key = 'resource_uri'\n\ + \ if uri_key not in resource:\n uri_key = 'resourceUri'\n\n gcp_resources_split\ + \ = resource[uri_key].partition('projects')\n resource_name = gcp_resources_split[1]\ + \ + gcp_resources_split[2]\n prefix_str = gcp_resources_split[0]\n prefix_str\ + \ = prefix_str[: prefix_str.find(api_endpoint_suffix)]\n api_endpoint =\ + \ (\n prefix_str[(prefix_str.rfind('//') + 2) :] + api_endpoint_suffix\n\ + \ )\n\n job_client = aip.JobServiceClient(\n client_options={'api_endpoint':\ + \ api_endpoint}\n )\n response = job_client.get_hyperparameter_tuning_job(\n\ + \ name=resource_name,\n retry=Retry(initial=10.0, maximum=60.0,\ + \ deadline=10.0 * 60.0),\n )\n\n # Get best trial\n trials_list = []\n\ + \ for trial in response.trials:\n if trial.final_measurement:\n \ + \ trials_list.append({\n 'id': trial.id,\n 'objective_value':\ + \ trial.final_measurement.metrics[0].value,\n })\n\n if study_spec_metric_goal\ + \ == 'MAXIMIZE':\n best_fn = max\n elif study_spec_metric_goal == 'MINIMIZE':\n\ + \ best_fn = min\n else:\n raise ValueError(\n f'Unexpected\ + \ study spec metric goal: {study_spec_metric_goal}'\n )\n\n best_trial\ + \ = best_fn(trials_list, key=lambda trial: trial['objective_value'])\n\n\ + \ # Build unmanaged_container_model\n unmanaged_container_model.metadata['containerSpec']\ + \ = {\n 'imageUri': prediction_docker_uri,\n 'healthRoute': '/health',\n\ + \ 'predictRoute': '/predict',\n }\n unmanaged_container_model.metadata['predictSchemata']\ + \ = {\n 'instanceSchemaUri': instance_schema_uri,\n 'predictionSchemaUri':\ + \ prediction_schema_uri,\n }\n unmanaged_container_model.uri = os.path.join(\n\ + \ trials_dir, 'trial_{}'.format(best_trial['id']), 'model'\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-prediction-type-for-xgboost: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_type_for_xgboost + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_type_for_xgboost(objective: str) -> str:\n \"\ + \"\"Returns prediction_type given XGBoost training objective..\n\n Args:\n\ + \ objective: The XGBoost training objective\n\n Returns:\n A string.\ + \ One of 'regression' or 'classification'\n \"\"\"\n if objective.startswith('binary')\ + \ or objective.startswith('multi'):\n return 'classification'\n elif\ + \ objective.startswith('reg'):\n return 'regression'\n else:\n raise\ + \ ValueError(\n f'Unsupported XGBoost training objective: {objective}.\ + \ Must be one of'\n ' [reg:squarederror, reg:squaredlogerror, reg:logistic,\ + \ reg:gamma,'\n ' reg:tweedie, reg:pseudohubererror, binary:logistic,'\n\ + \ ' multi:softprob].'\n )\n\n" + image: python:3.7 + exec-get-xgboost-study-spec-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_xgboost_study_spec_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_xgboost_study_spec_parameters(\n study_spec_parameters_override:\ + \ list, # Required for KFP validation; pylint:disable=g-bare-generic,unused-argument\n\ + ) -> list: # Required for KFP validation; pylint:disable=g-bare-generic\n\ + \ \"\"\"Get study_spec_parameters for an XGBoost hyperparameter tuning\ + \ job.\n\n Args:\n study_spec_parameters_override: List of dictionaries\ + \ representing parameters\n to optimize. The dictionary key is the\ + \ parameter_id, which is passed to\n training job as a command line\ + \ argument, and the dictionary value is the\n parameter specification\ + \ of the metric.\n\n Returns:\n List of final Vizier study_spec_parameters\ + \ of type ParameterSpec.\n \"\"\"\n # pylint:disable=g-import-not-at-top,redefined-outer-name,reimported\n\ + \ import functools\n import math\n from typing import Any, Dict, List,\ + \ Optional\n # pylint:enable=g-import-not-at-top,redefined-outer-name,reimported\n\ + \n # Need to define constants within the component function\n # pylint:disable=invalid-name\n\ + \ _GBTREE_BOOSTER = 'gbtree'\n _GBLINEAR_BOOSTER = 'gblinear'\n _DART_BOOSTER\ + \ = 'dart'\n _XGBOOST_BOOSTER_PARAMETERS_MAP = {\n 'eta': [_GBTREE_BOOSTER,\ + \ _DART_BOOSTER],\n 'gamma': [_GBTREE_BOOSTER, _DART_BOOSTER],\n \ + \ 'max_depth': [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'min_child_weight':\ + \ [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'max_delta_step': [_GBTREE_BOOSTER,\ + \ _DART_BOOSTER],\n 'subsample': [_GBTREE_BOOSTER, _DART_BOOSTER],\n\ + \ 'colsample_bytree': [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'colsample_bylevel':\ + \ [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'colsample_bynode': [_GBTREE_BOOSTER,\ + \ _DART_BOOSTER],\n 'lambda': [_GBTREE_BOOSTER, _DART_BOOSTER, _GBLINEAR_BOOSTER],\n\ + \ 'alpha': [_GBTREE_BOOSTER, _DART_BOOSTER, _GBLINEAR_BOOSTER],\n \ + \ 'tree_method': [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'scale_pos_weight':\ + \ [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'updater': [_GBTREE_BOOSTER,\ + \ _DART_BOOSTER, _GBLINEAR_BOOSTER],\n 'refresh_leaf': [_GBTREE_BOOSTER,\ + \ _DART_BOOSTER],\n 'process_type': [_GBTREE_BOOSTER, _DART_BOOSTER],\n\ + \ 'grow_policy': [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'sampling_method':\ + \ [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'monotone_constraints': [_GBTREE_BOOSTER,\ + \ _DART_BOOSTER],\n 'interaction_constraints': [_GBTREE_BOOSTER, _DART_BOOSTER],\n\ + \ 'sample_type': [_DART_BOOSTER],\n 'normalize_type': [_DART_BOOSTER],\n\ + \ 'rate_drop': [_DART_BOOSTER],\n 'one_drop': [_DART_BOOSTER],\n\ + \ 'skip_drop': [_DART_BOOSTER],\n 'num_parallel_tree': [_GBLINEAR_BOOSTER],\n\ + \ 'feature_selector': [_GBLINEAR_BOOSTER],\n 'top_k': [_GBLINEAR_BOOSTER],\n\ + \ 'max_leaves': [_GBTREE_BOOSTER, _DART_BOOSTER],\n 'max_bin':\ + \ [_GBTREE_BOOSTER, _DART_BOOSTER],\n }\n _XGBOOST_NO_DEFAULT_BOOSTER_PARAMS\ + \ = frozenset(\n ['updater', 'monotone_constraints', 'interaction_constraints']\n\ + \ )\n\n def _validate_float_spec(\n parameter_spec: Dict[str, Any],\ + \ lower_bound: float, upper_bound: float\n ) -> None:\n msg = (\n \ + \ f'Parameter spec for {parameter_spec[\"parameter_id\"]} must contain\ + \ '\n 'double_value_spec or discrete_value_spec with float values\ + \ within '\n f'the range of {lower_bound} and {upper_bound} (inclusive)'\n\ + \ )\n if 'double_value_spec' in parameter_spec:\n float_spec\ + \ = parameter_spec['double_value_spec']\n if float_spec['min_value']\ + \ < lower_bound:\n raise ValueError(\n f'{msg}, but got\ + \ {float_spec[\"min_value\"]} for min_value.'\n )\n if float_spec['max_value']\ + \ > upper_bound:\n raise ValueError(\n f'{msg}, but got\ + \ {float_spec[\"max_value\"]} for max_value.'\n )\n elif 'discrete_value_spec'\ + \ in parameter_spec:\n float_spec = parameter_spec['discrete_value_spec']\n\ + \ float_values = float_spec['values']\n for val in float_values:\n\ + \ if val < lower_bound or val > upper_bound:\n raise ValueError(f'{msg},\ + \ but got {val} in {float_values}.')\n else:\n raise ValueError(\n\ + \ f'Unexpected value spec for {parameter_spec[\"parameter_id\"\ + ]}. {msg}.'\n )\n\n def _validate_int_spec(\n parameter_spec:\ + \ Dict[str, Any],\n lower_bound: Optional[int],\n upper_bound:\ + \ Optional[int],\n ) -> None:\n msg = (\n f'Parameter spec for\ + \ {parameter_spec[\"parameter_id\"]} must contain '\n 'integer_value_spec\ + \ or discrete_value_spec with integer values within '\n f'the range\ + \ of {lower_bound} and {upper_bound} (inclusive)'\n )\n if 'integer_value_spec'\ + \ in parameter_spec:\n int_spec = parameter_spec['integer_value_spec']\n\ + \ if lower_bound is not None and int_spec['min_value'] < lower_bound:\n\ + \ raise ValueError(\n f'{msg}, but got {int_spec[\"min_value\"\ + ]} for min_value.'\n )\n if upper_bound is not None and int_spec['max_value']\ + \ > upper_bound:\n raise ValueError(\n f'{msg}, but got\ + \ {int_spec[\"max_value\"]} for max_value.'\n )\n elif 'discrete_value_spec'\ + \ in parameter_spec:\n int_values = parameter_spec['discrete_value_spec']['values']\n\ + \ for val in int_values:\n if not isinstance(val, int):\n \ + \ raise ValueError(\n f'{msg}, but got non-integer {val}\ + \ with '\n f'type {type(val)} in {int_values}.'\n \ + \ )\n if (lower_bound is not None and val < lower_bound) or (\n \ + \ upper_bound is not None and val > upper_bound\n ):\n\ + \ raise ValueError(f'{msg}, but got {val} in {int_values}.')\n\ + \ else:\n raise ValueError(\n f'Unexpected value spec for\ + \ {parameter_spec[\"parameter_id\"]}. {msg}.'\n )\n\n def _validate_categorical_spec(\n\ + \ parameter_spec: Dict[str, Any], valid_categories: Optional[List[str]]\n\ + \ ) -> None:\n msg = (\n f'Parameter spec for {parameter_spec[\"\ + parameter_id\"]} must contain '\n 'categorical_value_spec with unique\ + \ categories from '\n f'{valid_categories}'\n )\n if 'categorical_value_spec'\ + \ in parameter_spec:\n if valid_categories is None:\n # Any\ + \ category is valid.\n return\n categorical_values = parameter_spec['categorical_value_spec']['values']\n\ + \ valid_categorical_values = set(categorical_values).intersection(\n\ + \ set(valid_categories)\n )\n if len(valid_categorical_values)\ + \ != len(categorical_values):\n raise ValueError(f'{msg}, but got\ + \ {categorical_values}.')\n else:\n raise ValueError(\n \ + \ f'Unexpected value spec for {parameter_spec[\"parameter_id\"]}. {msg}.'\n\ + \ )\n\n _XGBOOST_PARAM_VALIDATIONS = {\n 'num_boost_round': functools.partial(\n\ + \ _validate_int_spec, lower_bound=1, upper_bound=None\n ),\n\ + \ 'early_stopping_rounds': functools.partial(\n _validate_int_spec,\ + \ lower_bound=1, upper_bound=None\n ),\n 'base_score': functools.partial(\n\ + \ _validate_float_spec, lower_bound=0, upper_bound=1\n ),\n\ + \ 'booster': functools.partial(\n _validate_categorical_spec,\n\ + \ valid_categories=['gbtree', 'gblinear', 'dart'],\n ),\n\ + \ 'eta': functools.partial(\n _validate_float_spec, lower_bound=0,\ + \ upper_bound=1\n ),\n 'gamma': functools.partial(\n \ + \ _validate_float_spec, lower_bound=0, upper_bound=math.inf\n ),\n\ + \ 'max_depth': functools.partial(\n _validate_int_spec, lower_bound=0,\ + \ upper_bound=None\n ),\n 'min_child_weight': functools.partial(\n\ + \ _validate_float_spec, lower_bound=0, upper_bound=math.inf\n \ + \ ),\n 'max_delta_step': functools.partial(\n _validate_float_spec,\ + \ lower_bound=0, upper_bound=math.inf\n ),\n 'subsample': functools.partial(\n\ + \ _validate_float_spec, lower_bound=0.0001, upper_bound=1\n \ + \ ),\n 'colsample_bytree': functools.partial(\n _validate_float_spec,\ + \ lower_bound=0.0001, upper_bound=1\n ),\n 'colsample_bylevel':\ + \ functools.partial(\n _validate_float_spec, lower_bound=0.0001,\ + \ upper_bound=1\n ),\n 'colsample_bynode': functools.partial(\n\ + \ _validate_float_spec, lower_bound=0.0001, upper_bound=1\n \ + \ ),\n 'lambda': functools.partial(\n _validate_float_spec,\ + \ lower_bound=0, upper_bound=1\n ),\n 'alpha': functools.partial(\n\ + \ _validate_float_spec, lower_bound=0, upper_bound=1\n ),\n\ + \ 'tree_method': functools.partial(\n _validate_categorical_spec,\n\ + \ valid_categories=['auto', 'exact', 'approx', 'hist', 'gpu_hist'],\n\ + \ ),\n 'scale_pos_weight': functools.partial(\n _validate_float_spec,\ + \ lower_bound=0, upper_bound=math.inf\n ),\n 'updater': functools.partial(\n\ + \ _validate_categorical_spec, valid_categories=None\n ),\n\ + \ 'refresh_leaf': functools.partial(\n _validate_int_spec,\ + \ lower_bound=0, upper_bound=1\n ),\n 'process_type': functools.partial(\n\ + \ _validate_categorical_spec, valid_categories=['default', 'updated']\n\ + \ ),\n 'grow_policy': functools.partial(\n _validate_categorical_spec,\n\ + \ valid_categories=['depthwise', 'lossguide'],\n ),\n \ + \ 'sampling_method': functools.partial(\n _validate_categorical_spec,\n\ + \ valid_categories=['uniform', 'gradient_based'],\n ),\n \ + \ 'monotone_constraints': functools.partial(\n _validate_categorical_spec,\ + \ valid_categories=None\n ),\n 'interaction_constraints': functools.partial(\n\ + \ _validate_categorical_spec, valid_categories=None\n ),\n\ + \ 'sample_type': functools.partial(\n _validate_categorical_spec,\ + \ valid_categories=['uniform', 'weighted']\n ),\n 'normalize_type':\ + \ functools.partial(\n _validate_categorical_spec, valid_categories=['tree',\ + \ 'forest']\n ),\n 'rate_drop': functools.partial(\n \ + \ _validate_float_spec, lower_bound=0, upper_bound=1\n ),\n 'one_drop':\ + \ functools.partial(\n _validate_int_spec, lower_bound=0, upper_bound=1\n\ + \ ),\n 'skip_drop': functools.partial(\n _validate_float_spec,\ + \ lower_bound=0, upper_bound=1\n ),\n 'num_parallel_tree': functools.partial(\n\ + \ _validate_int_spec, lower_bound=1, upper_bound=None\n ),\n\ + \ 'feature_selector': functools.partial(\n _validate_categorical_spec,\n\ + \ valid_categories=['cyclic', 'shuffle', 'random', 'greedy', 'thrifty'],\n\ + \ ),\n 'top_k': functools.partial(\n _validate_int_spec,\ + \ lower_bound=0, upper_bound=None\n ),\n 'max_cat_to_onehot':\ + \ functools.partial(\n _validate_int_spec, lower_bound=0, upper_bound=None\n\ + \ ),\n 'max_leaves': functools.partial(\n _validate_int_spec,\ + \ lower_bound=0, upper_bound=None\n ),\n 'max_bin': functools.partial(\n\ + \ _validate_int_spec, lower_bound=0, upper_bound=None\n ),\n\ + \ }\n\n def _add_booster_param(\n override_booster_params: Dict[str,\ + \ Any],\n param: Dict[str, Any],\n override_boosters: List[str],\n\ + \ ) -> None:\n # Validate parameter spec.\n param_id = param['parameter_spec']['parameter_id']\n\ + \ validation_func = _XGBOOST_PARAM_VALIDATIONS[param_id]\n validation_func(param['parameter_spec'])\n\ + \ # Add parameter spec for valid boosters.\n parent_boosters = param['parent_categorical_values']['values']\n\ + \ all_boosters = set(_XGBOOST_BOOSTER_PARAMETERS_MAP[param_id]).intersection(\n\ + \ set(override_boosters)\n )\n valid_parent_boosters = set(parent_boosters).intersection(all_boosters)\n\ + \ if valid_parent_boosters:\n override_booster_params[param_id]\ + \ = {}\n for booster in valid_parent_boosters:\n override_booster_params[param_id][booster]\ + \ = param['parameter_spec']\n\n def _get_booster_param_specs(\n override_booster_params:\ + \ Dict[str, Any],\n param_id: str,\n default_param_spec: Optional[Dict[str,\ + \ Any]],\n ) -> List[Dict[str, Any]]:\n if param_id not in override_booster_params:\n\ + \ if default_param_spec is None:\n return []\n return [default_param_spec]\n\ + \ override_param_specs = override_booster_params[param_id]\n if default_param_spec\ + \ is not None:\n for booster in default_param_spec['parent_categorical_values']['values']:\n\ + \ if booster not in override_param_specs:\n override_param_specs[booster]\ + \ = default_param_spec['parameter_spec']\n param_specs = []\n for\ + \ booster, override_spec in override_param_specs.items():\n included\ + \ = False\n for spec in param_specs:\n if spec['parameter_spec']\ + \ == override_spec:\n spec['parent_categorical_values']['values'].append(booster)\n\ + \ included = True\n break\n if not included:\n \ + \ param_specs.append({\n 'parameter_spec': override_spec,\n\ + \ 'parent_categorical_values': {'values': [booster]},\n \ + \ })\n return param_specs\n\n default_params = [\n {\n \ + \ 'parameter_id': 'num_boost_round',\n 'discrete_value_spec':\ + \ {'values': [1, 5, 10, 15, 20]},\n },\n {\n 'parameter_id':\ + \ 'early_stopping_rounds',\n 'discrete_value_spec': {'values':\ + \ [3, 5, 10]},\n },\n {'parameter_id': 'base_score', 'discrete_value_spec':\ + \ {'values': [0.5]}},\n {\n 'parameter_id': 'booster',\n \ + \ 'categorical_value_spec': {'values': ['gbtree', 'gblinear', 'dart']},\n\ + \ 'conditional_parameter_specs': [\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'eta',\n\ + \ 'double_value_spec': {\n \ + \ 'min_value': 0.0001,\n 'max_value': 1.0,\n \ + \ },\n 'scale_type': 'UNIT_LOG_SCALE',\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'gamma',\n\ + \ 'discrete_value_spec': {\n \ + \ 'values': [0, 10, 50, 100, 500, 1000]\n },\n \ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'max_depth',\n\ + \ 'integer_value_spec': {'min_value': 6, 'max_value':\ + \ 10},\n 'scale_type': 'UNIT_LINEAR_SCALE',\n \ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'min_child_weight',\n\ + \ 'double_value_spec': {\n \ + \ 'min_value': 0.0,\n 'max_value': 10.0,\n \ + \ },\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'max_delta_step',\n\ + \ 'discrete_value_spec': {\n \ + \ 'values': [0.0, 1.0, 3.0, 5.0, 7.0, 9.0]\n },\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'subsample',\n\ + \ 'double_value_spec': {\n \ + \ 'min_value': 0.0001,\n 'max_value': 1.0,\n \ + \ },\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'colsample_bytree',\n\ + \ 'double_value_spec': {\n \ + \ 'min_value': 0.0001,\n 'max_value': 1.0,\n \ + \ },\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'colsample_bylevel',\n\ + \ 'double_value_spec': {\n \ + \ 'min_value': 0.0001,\n 'max_value': 1.0,\n \ + \ },\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'colsample_bynode',\n\ + \ 'double_value_spec': {\n \ + \ 'min_value': 0.0001,\n 'max_value': 1.0,\n \ + \ },\n 'scale_type': 'UNIT_LINEAR_SCALE',\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'lambda',\n\ + \ 'double_value_spec': {\n \ + \ 'min_value': 0.0001,\n 'max_value': 1.0,\n \ + \ },\n 'scale_type': 'UNIT_REVERSE_LOG_SCALE',\n\ + \ },\n 'parent_categorical_values': {\n\ + \ 'values': ['gbtree', 'dart', 'gblinear']\n \ + \ },\n },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'alpha',\n \ + \ 'double_value_spec': {\n 'min_value': 0.0001,\n\ + \ 'max_value': 1.0,\n },\n\ + \ 'scale_type': 'UNIT_LOG_SCALE',\n \ + \ },\n 'parent_categorical_values': {\n \ + \ 'values': ['gbtree', 'dart', 'gblinear']\n },\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'tree_method',\n \ + \ 'categorical_value_spec': {'values': ['auto']},\n \ + \ },\n 'parent_categorical_values': {'values': ['gbtree',\ + \ 'dart']},\n },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'scale_pos_weight',\n \ + \ 'discrete_value_spec': {'values': [1.0]},\n \ + \ },\n 'parent_categorical_values': {'values': ['gbtree',\ + \ 'dart']},\n },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'refresh_leaf',\n \ + \ 'discrete_value_spec': {'values': [1]},\n },\n\ + \ 'parent_categorical_values': {'values': ['gbtree', 'dart']},\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'process_type',\n \ + \ 'categorical_value_spec': {'values': ['default']},\n \ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'grow_policy',\n\ + \ 'categorical_value_spec': {'values': ['depthwise']},\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'sampling_method',\n\ + \ 'categorical_value_spec': {'values': ['uniform']},\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gbtree', 'dart']},\n },\n {\n \ + \ 'parameter_spec': {\n 'parameter_id': 'sample_type',\n\ + \ 'categorical_value_spec': {'values': ['uniform']},\n\ + \ },\n 'parent_categorical_values': {'values':\ + \ ['dart']},\n },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'normalize_type',\n \ + \ 'categorical_value_spec': {'values': ['tree']},\n \ + \ },\n 'parent_categorical_values': {'values':\ + \ ['dart']},\n },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'rate_drop',\n \ + \ 'discrete_value_spec': {'values': [0.0]},\n },\n\ + \ 'parent_categorical_values': {'values': ['dart']},\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'one_drop',\n \ + \ 'discrete_value_spec': {'values': [0]},\n },\n\ + \ 'parent_categorical_values': {'values': ['dart']},\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'skip_drop',\n \ + \ 'discrete_value_spec': {'values': [0.0]},\n },\n\ + \ 'parent_categorical_values': {'values': ['dart']},\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'num_parallel_tree',\n \ + \ 'discrete_value_spec': {'values': [1]},\n \ + \ },\n 'parent_categorical_values': {'values': ['gblinear']},\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'feature_selector',\n \ + \ 'categorical_value_spec': {'values': ['cyclic']},\n \ + \ },\n 'parent_categorical_values': {'values':\ + \ ['gblinear']},\n },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'top_k',\n \ + \ 'discrete_value_spec': {'values': [0]},\n },\n \ + \ 'parent_categorical_values': {'values': ['gblinear']},\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'max_leaves',\n \ + \ 'discrete_value_spec': {'values': [0]},\n },\n\ + \ 'parent_categorical_values': {'values': ['gbtree', 'dart']},\n\ + \ },\n {\n 'parameter_spec':\ + \ {\n 'parameter_id': 'max_bin',\n \ + \ 'discrete_value_spec': {'values': [256]},\n },\n\ + \ 'parent_categorical_values': {'values': ['gbtree', 'dart']},\n\ + \ },\n ],\n },\n ]\n\n # Construct dictionaries\ + \ so that parameter specs are accessible by id.\n override_params = {}\n\ + \ override_booster_params = {}\n for param in study_spec_parameters_override:\n\ + \ # Validate a study spec before adding to the override_params dictionary.\n\ + \ validation_func = _XGBOOST_PARAM_VALIDATIONS[param['parameter_id']]\n\ + \ validation_func(param)\n override_params[param['parameter_id']]\ + \ = param\n\n # Add any param that does not have a default parameter\ + \ spec.\n if (\n param['parameter_id'] == 'max_cat_to_onehot'\n\ + \ and param['parameter_id'] not in default_params\n ):\n \ + \ default_params.append(param)\n if (\n param['parameter_id']\ + \ == 'booster'\n and 'conditional_parameter_specs' in param\n \ + \ ):\n for booster_param in param['conditional_parameter_specs']:\n\ + \ _add_booster_param(\n override_booster_params,\n \ + \ booster_param,\n override_boosters=param['categorical_value_spec']['values'],\n\ + \ )\n\n # Validate override params according to XGBoost param dependencies.\n\ + \ tree_method = override_booster_params.get('tree_method', None)\n if\ + \ tree_method is not None:\n for booster, tree_method_spec in tree_method.items():\n\ + \ if tree_method_spec['categorical_value_spec']['values'] != ['exact']:\n\ + \ continue\n # TODO(b/277777886): exact requires non-zero max_depth\ + \ value.\n # The below code is no longer necessary after raising min_value\ + \ to 6 in\n # the default spec. In the long run, we need to decide\ + \ the best\n # approach for max_depth. Keeping the code for now in\ + \ case the approach\n # involves overriding max_depth for 'exact' tree_method.\n\ + \ max_depth_spec = {\n 'parameter_id': 'max_depth',\n \ + \ 'integer_value_spec': {'min_value': 6, 'max_value': 10},\n \ + \ 'scale_type': 'UNIT_LINEAR_SCALE',\n }\n override_booster_params['max_depth']\ + \ = override_booster_params.get(\n 'max_depth', {booster: max_depth_spec}\n\ + \ )\n override_booster_params['max_depth'][booster] = override_booster_params[\n\ + \ 'max_depth'\n ].get(booster, max_depth_spec)\n try:\n\ + \ _validate_int_spec(\n override_booster_params['max_depth'][booster],\n\ + \ lower_bound=1,\n upper_bound=None,\n )\n\ + \ except ValueError as e:\n raise ValueError(\n 'max_depth\ + \ cannot be 0 (or < 0) when tree method is fixed to be '\n '\"\ + exact\".'\n ) from e\n\n # Construct the modified study specs study_spec_parameters.\n\ + \ study_spec_parameters = []\n for default_param in default_params:\n\ + \ override_param = override_params.get(\n default_param['parameter_id'],\ + \ default_param\n )\n study_spec_parameters.append(override_param)\n\ + \ # Override conditional parameters for booster.\n if default_param['parameter_id']\ + \ == 'booster':\n booster_param_specs = []\n override_booster_vals\ + \ = override_param['categorical_value_spec']['values']\n\n for booster_param\ + \ in default_param['conditional_parameter_specs']:\n override_parent_boosters\ + \ = set(\n booster_param['parent_categorical_values']['values']\n\ + \ ).intersection(override_booster_vals)\n if not override_parent_boosters:\n\ + \ # No need to include a booster param if no relevant booster will\n\ + \ # be used.\n continue\n # Update default booster\ + \ param boosters to exclude irrelevant boosters.\n booster_param['parent_categorical_values']['values']\ + \ = list(\n override_parent_boosters\n )\n booster_param_specs.extend(\n\ + \ _get_booster_param_specs(\n override_booster_params,\n\ + \ param_id=booster_param['parameter_spec']['parameter_id'],\n\ + \ default_param_spec=booster_param,\n )\n \ + \ )\n\n for booster_param_name in _XGBOOST_NO_DEFAULT_BOOSTER_PARAMS:\n\ + \ booster_param_specs.extend(\n _get_booster_param_specs(\n\ + \ override_booster_params,\n param_id=booster_param_name,\n\ + \ default_param_spec=None,\n )\n )\n\n\ + \ # booster_param_specs combines the overriding booster parameter\n\ + \ # specs from user input and the default booster parameter specs.\n\ + \ override_param['conditional_parameter_specs'] = booster_param_specs\n\ + \n return study_spec_parameters\n\n" + image: python:3.7 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"pipeline_job\": + \"", "projects/{{$.inputs.parameters[''project'']}}/locations/{{$.inputs.parameters[''location'']}}/pipelineJobs/{{$.pipeline_job_uuid}}", + "\"", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.model.upload_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + exec-xgboost-hyperparameter-tuning-job: + container: + args: + - --type + - HyperparameterTuningJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"xgboost-hyperparameter-tuning-job-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"study_spec\": {\"metrics\": [{\"metric_id\": \"", "{{$.inputs.parameters[''study_spec_metric_id'']}}", + "\", \"goal\": \"", "{{$.inputs.parameters[''study_spec_metric_goal'']}}", + "\"}], \"parameters\": ", "{{$.inputs.parameters[''study_spec_parameters_override'']}}", + ", \"algorithm\": \"", "{{$.inputs.parameters[''study_spec_algorithm'']}}", + "\", \"measurement_selection_type\": \"", "{{$.inputs.parameters[''study_spec_measurement_selection_type'']}}", + "\"}, \"max_trial_count\": ", "{{$.inputs.parameters[''max_trial_count'']}}", + ", \"parallel_trial_count\": ", "{{$.inputs.parameters[''parallel_trial_count'']}}", + ", \"max_failed_trial_count\": ", "{{$.inputs.parameters[''max_failed_trial_count'']}}", + ", \"trial_job_spec\": {\"worker_pool_specs\": ", "{{$.inputs.parameters[''worker_pool_specs'']}}", + "}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.hyperparameter_tuning_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 +pipelineInfo: + description: The XGBoost HyperparameterTuningJob pipeline. + name: automl-tabular-xgboost-hyperparameter-tuning-job +root: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + pipelinechannel--dataset_level_transformations: + componentInputParameter: dataset_level_transformations + pipelinechannel--disable_default_eval_metric: + componentInputParameter: disable_default_eval_metric + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--eval_metric: + componentInputParameter: eval_metric + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--max_failed_trial_count: + componentInputParameter: max_failed_trial_count + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--max_trial_count: + componentInputParameter: max_trial_count + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--objective: + componentInputParameter: objective + pipelinechannel--parallel_trial_count: + componentInputParameter: parallel_trial_count + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--run_feature_selection: + componentInputParameter: run_feature_selection + pipelinechannel--seed: + componentInputParameter: seed + pipelinechannel--seed_per_iteration: + componentInputParameter: seed_per_iteration + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--study_spec_algorithm: + componentInputParameter: study_spec_algorithm + pipelinechannel--study_spec_measurement_selection_type: + componentInputParameter: study_spec_measurement_selection_type + pipelinechannel--study_spec_metric_goal: + componentInputParameter: study_spec_metric_goal + pipelinechannel--study_spec_metric_id: + componentInputParameter: study_spec_metric_id + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + pipelinechannel--tf_custom_transformation_definitions: + componentInputParameter: tf_custom_transformation_definitions + pipelinechannel--tf_transformations_path: + componentInputParameter: tf_transformations_path + pipelinechannel--training_accelerator_count: + componentInputParameter: training_accelerator_count + pipelinechannel--training_accelerator_type: + componentInputParameter: training_accelerator_type + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--training_machine_type: + componentInputParameter: training_machine_type + pipelinechannel--training_total_replica_count: + componentInputParameter: training_total_replica_count + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Parent model if this model is uploaded as a version. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'The BigQuery staging full dataset id for + + storing intermediate tables.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: The BigQuery data source. + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: The CSV data source. + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + description: 'Dataset-level custom + + transformation definitions in string format.' + isOptional: true + parameterType: LIST + dataset_level_transformations: + description: 'Dataset-level transformation configuration in + + string format.' + isOptional: true + parameterType: LIST + disable_default_eval_metric: + defaultValue: 0.0 + description: 'Flag to disable default metric. Set to >0 to + + disable. Default to 0.' + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eval_metric: + defaultValue: '' + description: 'Evaluation metrics for validation data represented as a + + comma-separated string.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_selection_algorithm: + defaultValue: AMI + description: Feature selection algorithm. + isOptional: true + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + max_failed_trial_count: + defaultValue: 0.0 + description: 'The number of failed trials that need to be seen + + before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides + + how many trials must fail before the whole job fails.' + isOptional: true + parameterType: NUMBER_INTEGER + max_selected_features: + defaultValue: -1.0 + description: Maximum number of features to select. + isOptional: true + parameterType: NUMBER_INTEGER + max_trial_count: + description: The desired total number of trials. + parameterType: NUMBER_INTEGER + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + objective: + description: 'Specifies the learning task and the learning objective. Must + be + + one of [reg:squarederror, reg:squaredlogerror, + + reg:logistic, reg:gamma, reg:tweedie, reg:pseudohubererror, + + binary:logistic, multi:softprob].' + parameterType: STRING + parallel_trial_count: + description: The desired number of trials to run in parallel. + parameterType: NUMBER_INTEGER + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: Whether to enable feature selection. + isOptional: true + parameterType: BOOLEAN + seed: + defaultValue: 0.0 + description: Random seed. + isOptional: true + parameterType: NUMBER_INTEGER + seed_per_iteration: + defaultValue: false + description: Seed PRNG determnisticly via iterator number. + isOptional: true + parameterType: BOOLEAN + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + study_spec_algorithm: + defaultValue: ALGORITHM_UNSPECIFIED + description: 'The search algorithm specified for the study. One of + + ''ALGORITHM_UNSPECIFIED'', ''GRID_SEARCH'', or ''RANDOM_SEARCH''.' + isOptional: true + parameterType: STRING + study_spec_measurement_selection_type: + defaultValue: BEST_MEASUREMENT + description: ' Which measurement to use if/when the + + service automatically selects the final measurement from previously + + reported intermediate measurements. One of "BEST_MEASUREMENT" or + + "LAST_MEASUREMENT".' + isOptional: true + parameterType: STRING + study_spec_metric_goal: + description: 'Optimization goal of the metric, possible values: + + "MAXIMIZE", "MINIMIZE".' + parameterType: STRING + study_spec_metric_id: + description: 'Metric to optimize. For options, please look under + + ''eval_metrics'' at + + https://xgboost.readthedocs.io/en/stable/parameter.html#learning-task-parameters.' + parameterType: STRING + study_spec_parameters_override: + description: 'List of dictionaries representing parameters + + to optimize. The dictionary key is the parameter_id, which is passed to + + training job as a command line argument, and the dictionary value is the + + parameter specification of the metric.' + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: Test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + description: 'List of auto transform features in the + + comma-separated string format.' + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + description: 'TF custom transformation definitions + + in string format.' + isOptional: true + parameterType: LIST + tf_transformations_path: + defaultValue: '' + description: Path to TF transformation configuration. + isOptional: true + parameterType: STRING + training_accelerator_count: + defaultValue: 0.0 + description: Accelerator count. + isOptional: true + parameterType: NUMBER_INTEGER + training_accelerator_type: + defaultValue: '' + description: Accelerator type. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + training_machine_type: + defaultValue: c2-standard-16 + description: Machine type. + isOptional: true + parameterType: STRING + training_total_replica_count: + defaultValue: 1.0 + description: Number of workers. + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + validation_fraction: + defaultValue: -1.0 + description: Validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer.py new file mode 100644 index 0000000000..582efa4dc3 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer.py @@ -0,0 +1,76 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML XGBoost Trainer component spec.""" + +from typing import Optional + +from kfp import dsl + + +@dsl.container_component +def xgboost_trainer( + project: str, + location: str, + worker_pool_specs: list, + gcp_resources: dsl.OutputPath(str), + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Trains an XGBoost model using Vertex CustomJob API. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + worker_pool_specs: The worker pool specs. + encryption_spec_key_name: The KMS key name. + + Returns: + gcp_resources: Serialized gcp_resources proto tracking the custom training job. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "xgboost-trainer-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + '"}, "job_spec": {"worker_pool_specs": ', + worker_pool_specs, + '}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml new file mode 100644 index 0000000000..1f761644b8 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/tabular/xgboost_trainer_pipeline.yaml @@ -0,0 +1,4470 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-xgboost-trainer +# Description: The XGBoost training pipeline. +# Inputs: +# base_score: float [Default: 0.5] +# bigquery_staging_full_dataset_id: str [Default: ''] +# booster: str [Default: 'gbtree'] +# colsample_bylevel: float [Default: 1.0] +# colsample_bynode: float [Default: 1.0] +# colsample_bytree: float [Default: 1.0] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# dataset_level_custom_transformation_definitions: list +# dataset_level_transformations: list +# disable_default_eval_metric: int [Default: 0.0] +# early_stopping_rounds: int [Default: -1.0] +# encryption_spec_key_name: str [Default: ''] +# eta: float [Default: 0.3] +# eval_metric: str [Default: ''] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# feature_selection_algorithm: str [Default: 'AMI'] +# feature_selector: str [Default: 'cyclic'] +# gamma: float [Default: 0.0] +# grow_policy: str [Default: 'depthwise'] +# huber_slope: float [Default: 1.0] +# interaction_constraints: str [Default: ''] +# location: str +# max_bin: int [Default: 256.0] +# max_cat_to_onehot: int [Default: -1.0] +# max_delta_step: float [Default: 0.0] +# max_depth: int [Default: 6.0] +# max_leaves: int [Default: 0.0] +# max_selected_features: int [Default: -1.0] +# min_child_weight: float [Default: 1.0] +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# monotone_constraints: str [Default: ''] +# normalize_type: str [Default: 'tree'] +# num_boost_round: int [Default: 10.0] +# num_parallel_tree: int [Default: 1.0] +# objective: str +# one_drop: int [Default: 0.0] +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# process_type: str [Default: 'default'] +# project: str +# rate_drop: float [Default: 0.0] +# refresh_leaf: int [Default: 1.0] +# reg_alpha: float [Default: 0.0] +# reg_lambda: float [Default: 1.0] +# root_dir: str +# run_evaluation: bool [Default: True] +# run_feature_selection: bool [Default: False] +# sample_type: str [Default: 'uniform'] +# sampling_method: str [Default: 'uniform'] +# scale_pos_weight: float [Default: 1.0] +# seed: int [Default: 0.0] +# seed_per_iteration: bool [Default: False] +# skip_drop: float [Default: 0.0] +# stratified_split_key: str [Default: ''] +# subsample: float [Default: 1.0] +# target_column: str +# test_fraction: float [Default: -1.0] +# tf_auto_transform_features: dict +# tf_custom_transformation_definitions: list +# tf_transformations_path: str [Default: ''] +# top_k: int [Default: 0.0] +# training_accelerator_count: int [Default: 0.0] +# training_accelerator_type: str [Default: ''] +# training_fraction: float [Default: -1.0] +# training_machine_type: str [Default: 'c2-standard-16'] +# training_total_replica_count: int [Default: 1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# tree_method: str [Default: 'auto'] +# tweedie_variance_power: float [Default: 1.5] +# updater: str [Default: ''] +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# Outputs: +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--generate-xgboost-trainer-worker-pool-specs-unmanaged_container_model + parameters: + bigquery_source_input_uri: + componentInputParameter: pipelinechannel--feature-transform-engine-bigquery_test_split_uri + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--get-prediction-type-for-xgboost-Output + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + inputDefinitions: + artifacts: + pipelinechannel--generate-xgboost-trainer-worker-pool-specs-unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + parameterType: STRING + pipelinechannel--get-prediction-type-for-xgboost-Output: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + dependentTasks: + - xgboost-trainer + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - bool-identity + - feature-transform-engine + - generate-xgboost-trainer-worker-pool-specs + - get-prediction-type-for-xgboost + inputs: + artifacts: + pipelinechannel--generate-xgboost-trainer-worker-pool-specs-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: generate-xgboost-trainer-worker-pool-specs + parameters: + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--feature-transform-engine-bigquery_test_split_uri: + taskOutputParameter: + outputParameterKey: bigquery_test_split_uri + producerTask: feature-transform-engine + pipelinechannel--get-prediction-type-for-xgboost-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-type-for-xgboost + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + dependentTasks: + - get-prediction-type-for-xgboost + inputs: + parameters: + bigquery_staging_full_dataset_id: + componentInputParameter: pipelinechannel--bigquery_staging_full_dataset_id + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataset_level_custom_transformation_definitions: + componentInputParameter: pipelinechannel--dataset_level_custom_transformation_definitions + dataset_level_transformations: + componentInputParameter: pipelinechannel--dataset_level_transformations + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + feature_selection_algorithm: + componentInputParameter: pipelinechannel--feature_selection_algorithm + location: + componentInputParameter: pipelinechannel--location + max_selected_features: + componentInputParameter: pipelinechannel--max_selected_features + model_type: + runtimeValue: + constant: boosted_trees + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-type-for-xgboost + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_feature_selection: + componentInputParameter: pipelinechannel--run_feature_selection + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + componentInputParameter: pipelinechannel--tf_auto_transform_features + tf_custom_transformation_definitions: + componentInputParameter: pipelinechannel--tf_custom_transformation_definitions + tf_transformations_path: + componentInputParameter: pipelinechannel--tf_transformations_path + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: feature-transform-engine + generate-xgboost-trainer-worker-pool-specs: + cachingOptions: + enableCache: true + componentRef: + name: comp-generate-xgboost-trainer-worker-pool-specs + dependentTasks: + - feature-transform-engine + - split-materialized-data + - training-configurator-and-validator + inputs: + artifacts: + instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: training-configurator-and-validator + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: split-materialized-data + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: split-materialized-data + training_schema_uri: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: feature-transform-engine + parameters: + accelerator_count: + componentInputParameter: pipelinechannel--training_accelerator_count + accelerator_type: + componentInputParameter: pipelinechannel--training_accelerator_type + base_score: + componentInputParameter: pipelinechannel--base_score + booster: + componentInputParameter: pipelinechannel--booster + colsample_bylevel: + componentInputParameter: pipelinechannel--colsample_bylevel + colsample_bynode: + componentInputParameter: pipelinechannel--colsample_bynode + colsample_bytree: + componentInputParameter: pipelinechannel--colsample_bytree + disable_default_eval_metric: + componentInputParameter: pipelinechannel--disable_default_eval_metric + early_stopping_rounds: + componentInputParameter: pipelinechannel--early_stopping_rounds + eta: + componentInputParameter: pipelinechannel--eta + eval_metric: + componentInputParameter: pipelinechannel--eval_metric + feature_selector: + componentInputParameter: pipelinechannel--feature_selector + gamma: + componentInputParameter: pipelinechannel--gamma + grow_policy: + componentInputParameter: pipelinechannel--grow_policy + huber_slope: + componentInputParameter: pipelinechannel--huber_slope + interaction_constraints: + componentInputParameter: pipelinechannel--interaction_constraints + machine_type: + componentInputParameter: pipelinechannel--training_machine_type + max_bin: + componentInputParameter: pipelinechannel--max_bin + max_cat_to_onehot: + componentInputParameter: pipelinechannel--max_cat_to_onehot + max_delta_step: + componentInputParameter: pipelinechannel--max_delta_step + max_depth: + componentInputParameter: pipelinechannel--max_depth + max_leaves: + componentInputParameter: pipelinechannel--max_leaves + min_child_weight: + componentInputParameter: pipelinechannel--min_child_weight + monotone_constraints: + componentInputParameter: pipelinechannel--monotone_constraints + normalize_type: + componentInputParameter: pipelinechannel--normalize_type + num_boost_round: + componentInputParameter: pipelinechannel--num_boost_round + num_parallel_tree: + componentInputParameter: pipelinechannel--num_parallel_tree + objective: + componentInputParameter: pipelinechannel--objective + one_drop: + componentInputParameter: pipelinechannel--one_drop + process_type: + componentInputParameter: pipelinechannel--process_type + rate_drop: + componentInputParameter: pipelinechannel--rate_drop + refresh_leaf: + componentInputParameter: pipelinechannel--refresh_leaf + reg_alpha: + componentInputParameter: pipelinechannel--reg_alpha + reg_lambda: + componentInputParameter: pipelinechannel--reg_lambda + sample_type: + componentInputParameter: pipelinechannel--sample_type + sampling_method: + componentInputParameter: pipelinechannel--sampling_method + scale_pos_weight: + componentInputParameter: pipelinechannel--scale_pos_weight + seed: + componentInputParameter: pipelinechannel--seed + seed_per_iteration: + componentInputParameter: pipelinechannel--seed_per_iteration + skip_drop: + componentInputParameter: pipelinechannel--skip_drop + subsample: + componentInputParameter: pipelinechannel--subsample + target_column: + componentInputParameter: pipelinechannel--target_column + top_k: + componentInputParameter: pipelinechannel--top_k + total_replica_count: + componentInputParameter: pipelinechannel--training_total_replica_count + tree_method: + componentInputParameter: pipelinechannel--tree_method + tweedie_variance_power: + componentInputParameter: pipelinechannel--tweedie_variance_power + updater: + componentInputParameter: pipelinechannel--updater + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: generate-xgboost-trainer-worker-pool-specs + get-prediction-type-for-xgboost: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-prediction-type-for-xgboost + inputs: + parameters: + objective: + componentInputParameter: pipelinechannel--objective + taskInfo: + name: get-prediction-type-for-xgboost + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - generate-xgboost-trainer-worker-pool-specs + - xgboost-trainer + inputs: + artifacts: + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: generate-xgboost-trainer-worker-pool-specs + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + split-materialized-data: + cachingOptions: + enableCache: true + componentRef: + name: comp-split-materialized-data + dependentTasks: + - feature-transform-engine + inputs: + artifacts: + materialized_data: + taskOutputArtifact: + outputArtifactKey: materialized_data + producerTask: feature-transform-engine + taskInfo: + name: split-materialized-data + training-configurator-and-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-training-configurator-and-validator + dependentTasks: + - feature-transform-engine + - get-prediction-type-for-xgboost + inputs: + artifacts: + dataset_stats: + taskOutputArtifact: + outputArtifactKey: dataset_stats + producerTask: feature-transform-engine + instance_schema: + taskOutputArtifact: + outputArtifactKey: instance_schema + producerTask: feature-transform-engine + training_schema: + taskOutputArtifact: + outputArtifactKey: training_schema + producerTask: feature-transform-engine + parameters: + prediction_type: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-prediction-type-for-xgboost + run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + split_example_counts: + taskOutputParameter: + outputParameterKey: split_example_counts + producerTask: feature-transform-engine + target_column: + componentInputParameter: pipelinechannel--target_column + weight_column: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: training-configurator-and-validator + xgboost-trainer: + cachingOptions: + enableCache: true + componentRef: + name: comp-xgboost-trainer + dependentTasks: + - generate-xgboost-trainer-worker-pool-specs + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + worker_pool_specs: + taskOutputParameter: + outputParameterKey: worker_pool_specs + producerTask: generate-xgboost-trainer-worker-pool-specs + taskInfo: + name: xgboost-trainer + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--base_score: + parameterType: NUMBER_DOUBLE + pipelinechannel--bigquery_staging_full_dataset_id: + parameterType: STRING + pipelinechannel--booster: + parameterType: STRING + pipelinechannel--colsample_bylevel: + parameterType: NUMBER_DOUBLE + pipelinechannel--colsample_bynode: + parameterType: NUMBER_DOUBLE + pipelinechannel--colsample_bytree: + parameterType: NUMBER_DOUBLE + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--dataset_level_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--dataset_level_transformations: + parameterType: LIST + pipelinechannel--disable_default_eval_metric: + parameterType: NUMBER_INTEGER + pipelinechannel--early_stopping_rounds: + parameterType: NUMBER_INTEGER + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--eta: + parameterType: NUMBER_DOUBLE + pipelinechannel--eval_metric: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--feature_selection_algorithm: + parameterType: STRING + pipelinechannel--feature_selector: + parameterType: STRING + pipelinechannel--gamma: + parameterType: NUMBER_DOUBLE + pipelinechannel--grow_policy: + parameterType: STRING + pipelinechannel--huber_slope: + parameterType: NUMBER_DOUBLE + pipelinechannel--interaction_constraints: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--max_bin: + parameterType: NUMBER_INTEGER + pipelinechannel--max_cat_to_onehot: + parameterType: NUMBER_INTEGER + pipelinechannel--max_delta_step: + parameterType: NUMBER_DOUBLE + pipelinechannel--max_depth: + parameterType: NUMBER_INTEGER + pipelinechannel--max_leaves: + parameterType: NUMBER_INTEGER + pipelinechannel--max_selected_features: + parameterType: NUMBER_INTEGER + pipelinechannel--min_child_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--monotone_constraints: + parameterType: STRING + pipelinechannel--normalize_type: + parameterType: STRING + pipelinechannel--num_boost_round: + parameterType: NUMBER_INTEGER + pipelinechannel--num_parallel_tree: + parameterType: NUMBER_INTEGER + pipelinechannel--objective: + parameterType: STRING + pipelinechannel--one_drop: + parameterType: NUMBER_INTEGER + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--process_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--rate_drop: + parameterType: NUMBER_DOUBLE + pipelinechannel--refresh_leaf: + parameterType: NUMBER_INTEGER + pipelinechannel--reg_alpha: + parameterType: NUMBER_DOUBLE + pipelinechannel--reg_lambda: + parameterType: NUMBER_DOUBLE + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--run_feature_selection: + parameterType: BOOLEAN + pipelinechannel--sample_type: + parameterType: STRING + pipelinechannel--sampling_method: + parameterType: STRING + pipelinechannel--scale_pos_weight: + parameterType: NUMBER_DOUBLE + pipelinechannel--seed: + parameterType: NUMBER_INTEGER + pipelinechannel--seed_per_iteration: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--skip_drop: + parameterType: NUMBER_DOUBLE + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--subsample: + parameterType: NUMBER_DOUBLE + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--tf_auto_transform_features: + parameterType: STRUCT + pipelinechannel--tf_custom_transformation_definitions: + parameterType: LIST + pipelinechannel--tf_transformations_path: + parameterType: STRING + pipelinechannel--top_k: + parameterType: NUMBER_INTEGER + pipelinechannel--training_accelerator_count: + parameterType: NUMBER_INTEGER + pipelinechannel--training_accelerator_type: + parameterType: STRING + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_machine_type: + parameterType: STRING + pipelinechannel--training_total_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--tree_method: + parameterType: STRING + pipelinechannel--tweedie_variance_power: + parameterType: NUMBER_DOUBLE + pipelinechannel--updater: + parameterType: STRING + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-generate-xgboost-trainer-worker-pool-specs: + executorLabel: exec-generate-xgboost-trainer-worker-pool-specs + inputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path to JSON file for baseline values. + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Required. The path to the materialized validation + + split.' + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Required. The path to the materialized train + + split.' + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Required. The path to the training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Required. The path to transform output. + parameters: + accelerator_count: + defaultValue: 0.0 + description: Accelerator count. + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: Accelerator type. + isOptional: true + parameterType: STRING + base_score: + defaultValue: 0.5 + description: The initial prediction score of all instances, global bias. + isOptional: true + parameterType: NUMBER_DOUBLE + booster: + defaultValue: gbtree + description: 'Which booster to use, can be gbtree, gblinear or dart. gbtree + and + + dart use tree based model while gblinear uses linear function.' + isOptional: true + parameterType: STRING + colsample_bylevel: + defaultValue: 1.0 + description: Subsample ratio of columns for each split, in each level. + isOptional: true + parameterType: NUMBER_DOUBLE + colsample_bynode: + defaultValue: 1.0 + description: Subsample ratio of columns for each node (split). + isOptional: true + parameterType: NUMBER_DOUBLE + colsample_bytree: + defaultValue: 1.0 + description: Subsample ratio of columns when constructing each tree. + isOptional: true + parameterType: NUMBER_DOUBLE + disable_default_eval_metric: + defaultValue: 0.0 + description: 'Flag to disable default metric. Set to >0 to + + disable. Default to 0.' + isOptional: true + parameterType: NUMBER_INTEGER + early_stopping_rounds: + defaultValue: -1.0 + description: 'Activates early stopping. Validation error needs to + + decrease at least every early_stopping_rounds round(s) to continue + + training.' + isOptional: true + parameterType: NUMBER_INTEGER + eta: + defaultValue: 0.3 + description: Learning rate. + isOptional: true + parameterType: NUMBER_DOUBLE + eval_metric: + defaultValue: '' + description: 'Evaluation metrics for validation data represented as a + + comma-separated string.' + isOptional: true + parameterType: STRING + feature_selector: + defaultValue: cyclic + description: '[linear booster only] Feature selection and ordering + + method.' + isOptional: true + parameterType: STRING + gamma: + defaultValue: 0.0 + description: 'Minimum loss reduction required to make a further partition + on a leaf + + node of the tree.' + isOptional: true + parameterType: NUMBER_DOUBLE + grow_policy: + defaultValue: depthwise + description: 'Controls a way new nodes are added to the tree. Only supported + + if tree_method is hist. Choices:["depthwise", "lossguide"]' + isOptional: true + parameterType: STRING + huber_slope: + defaultValue: 1.0 + description: 'A parameter used for Pseudo-Huber loss to define the delta + + term.' + isOptional: true + parameterType: NUMBER_DOUBLE + interaction_constraints: + defaultValue: '' + description: 'Constraints for interaction representing permitted + + interactions.' + isOptional: true + parameterType: STRING + machine_type: + defaultValue: c2-standard-16 + description: Machine type. + isOptional: true + parameterType: STRING + max_bin: + defaultValue: 256.0 + description: Maximum number of discrete bins to bucket continuous features. + isOptional: true + parameterType: NUMBER_INTEGER + max_cat_to_onehot: + defaultValue: -1.0 + description: 'A threshold for deciding whether XGBoost should use + + one-hot encoding based split for categorical data.' + isOptional: true + parameterType: NUMBER_INTEGER + max_delta_step: + defaultValue: 0.0 + description: 'Maximum delta step we allow each tree''s weight estimation + to + + be.' + isOptional: true + parameterType: NUMBER_DOUBLE + max_depth: + defaultValue: 6.0 + description: Maximum depth of a tree. + isOptional: true + parameterType: NUMBER_INTEGER + max_leaves: + defaultValue: 0.0 + description: Maximum number of nodes to be added. + isOptional: true + parameterType: NUMBER_INTEGER + min_child_weight: + defaultValue: 1.0 + description: Minimum sum of instance weight(hessian) needed in a child. + isOptional: true + parameterType: NUMBER_DOUBLE + monotone_constraints: + defaultValue: '' + description: Constraint of variable monotonicity. + isOptional: true + parameterType: STRING + normalize_type: + defaultValue: tree + description: '[dart booster only] Type of normalization algorithm, + + Choices:["tree", "forest"]' + isOptional: true + parameterType: STRING + num_boost_round: + defaultValue: 10.0 + description: Number of boosting iterations. + isOptional: true + parameterType: NUMBER_INTEGER + num_parallel_tree: + defaultValue: 1.0 + description: 'Number of parallel trees constructed during each + + iteration. This option is used to support boosted random forest.' + isOptional: true + parameterType: NUMBER_INTEGER + objective: + description: Required. Specifies the learning task and the learning objective. + parameterType: STRING + one_drop: + defaultValue: 0.0 + description: '[dart booster only] When this flag is enabled, at least one + tree + + is always dropped during the dropout (allows Binomial-plus-one or + + epsilon-dropout from the original DART paper).' + isOptional: true + parameterType: NUMBER_INTEGER + process_type: + defaultValue: default + description: 'A type of boosting process to run. Choices:["default", + + "update"]' + isOptional: true + parameterType: STRING + rate_drop: + defaultValue: 0.0 + description: '[dart booster only] Dropout rate.''' + isOptional: true + parameterType: NUMBER_DOUBLE + refresh_leaf: + defaultValue: 1.0 + description: 'Refresh updater plugin. Update tree leaf and nodes''s stats + if + + True. When it is False, only node stats are updated.' + isOptional: true + parameterType: NUMBER_INTEGER + reg_alpha: + defaultValue: 0.0 + description: L1 regularization term on weights. + isOptional: true + parameterType: NUMBER_DOUBLE + reg_lambda: + defaultValue: 1.0 + description: L2 regularization term on weights. + isOptional: true + parameterType: NUMBER_DOUBLE + sample_type: + defaultValue: uniform + description: '[dart booster only] Type of sampling algorithm. + + Choices:["uniform", "weighted"]' + isOptional: true + parameterType: STRING + sampling_method: + defaultValue: uniform + description: The method to use to sample the training instances. + isOptional: true + parameterType: STRING + scale_pos_weight: + defaultValue: 1.0 + description: Control the balance of positive and negative weights. + isOptional: true + parameterType: NUMBER_DOUBLE + seed: + defaultValue: 0.0 + description: Random seed. + isOptional: true + parameterType: NUMBER_INTEGER + seed_per_iteration: + defaultValue: false + description: Seed PRNG determnisticly via iterator number. + isOptional: true + parameterType: BOOLEAN + skip_drop: + defaultValue: 0.0 + description: '[dart booster only] Probability of skipping the dropout procedure + + during a boosting iteration.' + isOptional: true + parameterType: NUMBER_DOUBLE + subsample: + defaultValue: 1.0 + description: Subsample ratio of the training instance. + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + description: Required. Target column name. + parameterType: STRING + top_k: + defaultValue: 0.0 + description: 'The number of top features to select in greedy and thrifty + feature + + selector. The value of 0 means using all the features.' + isOptional: true + parameterType: NUMBER_INTEGER + total_replica_count: + description: Number of workers. + parameterType: NUMBER_INTEGER + tree_method: + defaultValue: auto + description: 'The tree construction algorithm used in XGBoost. Choices: + + ["auto", "exact", "approx", "hist", "gpu_exact", "gpu_hist"].' + isOptional: true + parameterType: STRING + tweedie_variance_power: + defaultValue: 1.5 + description: 'Parameter that controls the variance of the Tweedie + + distribution.' + isOptional: true + parameterType: NUMBER_DOUBLE + updater: + defaultValue: '' + description: 'A comma separated string defining the sequence of tree updaters + to + + run.' + isOptional: true + parameterType: STRING + weight_column: + defaultValue: '' + description: Weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + job_dir: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + unmanaged_container_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + worker_pool_specs: + parameterType: LIST + comp-get-prediction-type-for-xgboost: + executorLabel: exec-get-prediction-type-for-xgboost + inputDefinitions: + parameters: + objective: + description: The XGBoost training objective + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: An artifact of a model which to upload a new version to. Only + specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: "The unmanaged container model to be uploaded. The Model can\ + \ be passed from an upstream step or imported via a KFP `dsl.importer`.\n\ + :Examples:\n ::\n\n from kfp import dsl\n from google_cloud_pipeline_components.google_cloud_pipeline_components.types\ + \ import artifact_types\n\n importer_spec = dsl.importer(\n artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model',\n\ + \ artifact_class=artifact_types.UnmanagedContainerModel,\n metadata={\n\ + \ 'containerSpec': { 'imageUri':\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod'\n\ + \ }\n })" + isOptional: true + parameters: + description: + defaultValue: '' + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + isOptional: true + parameterType: STRING + display_name: + description: 'The display name of the Model. The name + + can be up to 128 characters long and can be consist of any UTF-8 + + characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model)' + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key spec for a Model. If set, this Model and all sub-resources of this + + Model will be secured by this key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + description: 'Metadata describing the Model''s + + input and output for explanation. Both `explanation_metadata` and `explanation_parameters` + must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata)' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters)' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your model. Label keys and values can be no longer than 64 + + characters (Unicode codepoints), can only contain lowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. See https://goo.gl/xmQnxf for more information and + + examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Optional location to upload this Model to. If + + not set, defaults to `us-central1`.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to upload this Model to. Defaults to the project in + which the PipelineJob is run. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: Artifact tracking the created Model. + parameters: + gcp_resources: + description: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) + which tracks the upload Model's long-running operation. + parameterType: STRING + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-split-materialized-data: + executorLabel: exec-split-materialized-data + inputDefinitions: + artifacts: + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'Materialized dataset output by the Feature + + Transform Engine.' + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized eval split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized test split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path patern to materialized train split. + comp-training-configurator-and-validator: + executorLabel: exec-training-configurator-and-validator + inputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Dataset stats generated by + + feature transform engine.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Schema of input data to the tf_model at + + serving time.' + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + available_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + available at forecast time.' + isOptional: true + parameterType: LIST + context_window: + defaultValue: -1.0 + description: The length of the context window. + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is + + enabled, the model will fit a distribution that captures the uncertainty + + of a prediction. At inference time, the predictive distribution is used + + to make a point prediction that minimizes the optimization objective. + + For example, the mean of a predictive distribution is the point + + prediction that minimizes RMSE loss. If quantiles are specified, then + + the quantiles of the distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + forecast_horizon: + defaultValue: -1.0 + description: The length of the forecast horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_model_type: + defaultValue: '' + description: The model types, e.g. l2l, seq2seq, tft. + isOptional: true + parameterType: STRING + forecasting_transformations: + defaultValue: {} + description: 'Dict mapping auto and/or type-resolutions to + + feature columns. The supported types are auto, categorical, numeric, + + text, and timestamp.' + isOptional: true + parameterType: STRUCT + group_columns: + description: 'A list of time series attribute column + + names that define the time series hierarchy.' + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over both the horizon and time series in the same + + hierarchy group.' + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over time series in the same group.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of "classification", + + "regression", "time_series".' + isOptional: true + parameterType: STRING + quantiles: + defaultValue: [] + description: All quantiles that the model need to predict. + isOptional: true + parameterType: LIST + run_distill: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: 'Whether we are running evaluation in the training + + pipeline.' + isOptional: true + parameterType: BOOLEAN + split_example_counts: + description: 'JSON string of data split example counts for + + train, validate, and test splits.' + parameterType: STRING + stage_1_deadline_hours: + description: 'Stage 1 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + stage_2_deadline_hours: + description: 'Stage 2 training budget in + + hours.' + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + description: 'The weight of the loss for + + predictions aggregated over the horizon for a single time series.' + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + defaultValue: '' + description: 'The column that indicates the time. Used by forecasting + + only.' + isOptional: true + parameterType: STRING + time_series_attribute_columns: + defaultValue: [] + description: 'The column names of the time series + + attributes.' + isOptional: true + parameterType: LIST + time_series_identifier_column: + description: '[Deprecated] The time series identifier + + column. Used by forecasting only. Raises exception if used - + + use the "time_series_identifier_column" field instead.' + isOptional: true + parameterType: STRING + time_series_identifier_columns: + defaultValue: [] + description: 'The list of time series identifier columns. + + Used by forecasting only.' + isOptional: true + parameterType: LIST + unavailable_at_forecast_columns: + defaultValue: [] + description: 'The names of the columns that are + + not available at forecast time.' + isOptional: true + parameterType: LIST + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + comp-xgboost-trainer: + executorLabel: exec-xgboost-trainer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + worker_pool_specs: + description: The worker pool specs. + parameterType: LIST + outputDefinitions: + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the custom training + + job.' + parameterType: STRING +deploymentSpec: + executors: + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 30.0 + exec-generate-xgboost-trainer-worker-pool-specs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _generate_xgboost_trainer_worker_pool_specs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _generate_xgboost_trainer_worker_pool_specs(\n total_replica_count:\ + \ int,\n target_column: str,\n objective: str,\n materialized_train_split:\ + \ dsl.InputPath('MaterializedSplit'),\n materialized_eval_split: dsl.InputPath('MaterializedSplit'),\n\ + \ transform_output: dsl.InputPath('TransformOutput'),\n training_schema_uri:\ + \ dsl.InputPath('DatasetSchema'),\n instance_baseline: dsl.InputPath('AutoMLTabularInstanceBaseline'),\n\ + \ job_dir: dsl.OutputPath('JobDir'),\n unmanaged_container_model:\ + \ dsl.Output[dsl.Artifact],\n machine_type: str = 'c2-standard-16',\n\ + \ accelerator_type: str = '',\n accelerator_count: int = 0,\n weight_column:\ + \ str = '',\n eval_metric: str = '',\n num_boost_round: int = 10,\n\ + \ early_stopping_rounds: int = -1,\n base_score: float = 0.5,\n \ + \ disable_default_eval_metric: int = 0,\n seed: int = 0,\n seed_per_iteration:\ + \ bool = False,\n booster: str = 'gbtree',\n eta: float = 0.3,\n \ + \ gamma: float = 0.0,\n max_depth: int = 6,\n min_child_weight:\ + \ float = 1.0,\n max_delta_step: float = 0.0,\n subsample: float =\ + \ 1.0,\n colsample_bytree: float = 1.0,\n colsample_bylevel: float\ + \ = 1.0,\n colsample_bynode: float = 1.0,\n reg_lambda: float = 1.0,\n\ + \ reg_alpha: float = 0.0,\n tree_method: str = 'auto',\n scale_pos_weight:\ + \ float = 1.0,\n updater: str = '',\n refresh_leaf: int = 1,\n \ + \ process_type: str = 'default',\n grow_policy: str = 'depthwise',\n\ + \ sampling_method: str = 'uniform',\n monotone_constraints: str =\ + \ '',\n interaction_constraints: str = '',\n sample_type: str = 'uniform',\n\ + \ normalize_type: str = 'tree',\n rate_drop: float = 0.0,\n one_drop:\ + \ int = 0,\n skip_drop: float = 0.0,\n num_parallel_tree: int = 1,\n\ + \ feature_selector: str = 'cyclic',\n top_k: int = 0,\n max_cat_to_onehot:\ + \ int = -1,\n max_leaves: int = 0,\n max_bin: int = 256,\n tweedie_variance_power:\ + \ float = 1.5,\n huber_slope: float = 1.0,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('worker_pool_specs', list), # pylint:disable=g-bare-generic\n\ + \ ],\n):\n \"\"\"Generates worker pool specs for XGBoost training.\n\ + \n For single machine XGBoost training, returns one worker pool spec for\ + \ master.\n For distributed XGBoost training, returns two worker pool specs,\ + \ the first one\n for master and the second one for the remaining workers.\n\ + \n Args:\n total_replica_count: Number of workers.\n target_column:\ + \ Required. Target column name.\n objective: Required. Specifies the\ + \ learning task and the learning objective.\n materialized_train_split:\ + \ Required. The path to the materialized train\n split.\n materialized_eval_split:\ + \ Required. The path to the materialized validation\n split.\n transform_output:\ + \ Required. The path to transform output.\n training_schema_uri: Required.\ + \ The path to the training schema.\n instance_baseline: Path to JSON\ + \ file for baseline values.\n job_dir: Job dir path.\n unmanaged_container_model:\ + \ The unmanaged model.\n machine_type: Machine type.\n accelerator_type:\ + \ Accelerator type.\n accelerator_count: Accelerator count.\n weight_column:\ + \ Weight column name.\n eval_metric: Evaluation metrics for validation\ + \ data represented as a\n comma-separated string.\n num_boost_round:\ + \ Number of boosting iterations.\n early_stopping_rounds: Activates early\ + \ stopping. Validation error needs to\n decrease at least every early_stopping_rounds\ + \ round(s) to continue\n training.\n base_score: The initial prediction\ + \ score of all instances, global bias.\n disable_default_eval_metric:\ + \ Flag to disable default metric. Set to >0 to\n disable. Default to\ + \ 0.\n seed: Random seed.\n seed_per_iteration: Seed PRNG determnisticly\ + \ via iterator number.\n booster: Which booster to use, can be gbtree,\ + \ gblinear or dart. gbtree and\n dart use tree based model while gblinear\ + \ uses linear function.\n eta: Learning rate.\n gamma: Minimum loss\ + \ reduction required to make a further partition on a leaf\n node of\ + \ the tree.\n max_depth: Maximum depth of a tree.\n min_child_weight:\ + \ Minimum sum of instance weight(hessian) needed in a child.\n max_delta_step:\ + \ Maximum delta step we allow each tree's weight estimation to\n be.\n\ + \ subsample: Subsample ratio of the training instance.\n colsample_bytree:\ + \ Subsample ratio of columns when constructing each tree.\n colsample_bylevel:\ + \ Subsample ratio of columns for each split, in each level.\n colsample_bynode:\ + \ Subsample ratio of columns for each node (split).\n reg_lambda: L2\ + \ regularization term on weights.\n reg_alpha: L1 regularization term\ + \ on weights.\n tree_method: The tree construction algorithm used in\ + \ XGBoost. Choices:\n [\"auto\", \"exact\", \"approx\", \"hist\", \"\ + gpu_exact\", \"gpu_hist\"].\n scale_pos_weight: Control the balance of\ + \ positive and negative weights.\n updater: A comma separated string\ + \ defining the sequence of tree updaters to\n run.\n refresh_leaf:\ + \ Refresh updater plugin. Update tree leaf and nodes's stats if\n True.\ + \ When it is False, only node stats are updated.\n process_type: A type\ + \ of boosting process to run. Choices:[\"default\",\n \"update\"]\n\ + \ grow_policy: Controls a way new nodes are added to the tree. Only supported\n\ + \ if tree_method is hist. Choices:[\"depthwise\", \"lossguide\"]\n\ + \ sampling_method: The method to use to sample the training instances.\n\ + \ monotone_constraints: Constraint of variable monotonicity.\n interaction_constraints:\ + \ Constraints for interaction representing permitted\n interactions.\n\ + \ sample_type: [dart booster only] Type of sampling algorithm.\n \ + \ Choices:[\"uniform\", \"weighted\"]\n normalize_type: [dart booster\ + \ only] Type of normalization algorithm,\n Choices:[\"tree\", \"forest\"\ + ]\n rate_drop: [dart booster only] Dropout rate.'\n one_drop: [dart\ + \ booster only] When this flag is enabled, at least one tree\n is always\ + \ dropped during the dropout (allows Binomial-plus-one or\n epsilon-dropout\ + \ from the original DART paper).\n skip_drop: [dart booster only] Probability\ + \ of skipping the dropout procedure\n during a boosting iteration.\n\ + \ num_parallel_tree: Number of parallel trees constructed during each\n\ + \ iteration. This option is used to support boosted random forest.\n\ + \ feature_selector: [linear booster only] Feature selection and ordering\n\ + \ method.\n top_k: The number of top features to select in greedy\ + \ and thrifty feature\n selector. The value of 0 means using all the\ + \ features.\n max_cat_to_onehot: A threshold for deciding whether XGBoost\ + \ should use\n one-hot encoding based split for categorical data.\n\ + \ max_leaves: Maximum number of nodes to be added.\n max_bin: Maximum\ + \ number of discrete bins to bucket continuous features.\n tweedie_variance_power:\ + \ Parameter that controls the variance of the Tweedie\n distribution.\n\ + \ huber_slope: A parameter used for Pseudo-Huber loss to define the delta\n\ + \ term.\n\n Raises:\n ValueError: If accelerator_count <= 0 and\ + \ accelerator_type is specified.\n\n Returns:\n Outputs containing the\ + \ worker pool specs.\n \"\"\"\n import copy\n import collections\n import\ + \ os\n import re\n\n def get_gcs_path(path):\n return re.sub(r'/gcs/',\ + \ 'gs://', path)\n\n formatted_job_dir = get_gcs_path(job_dir)\n prediction_docker_uri\ + \ = (\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/xgboost-prediction-server:20231002_0125'\n\ + \ )\n master_worker_pool_spec = {\n 'replica_count': 1,\n 'machine_spec':\ + \ {\n 'machine_type': machine_type,\n },\n 'container_spec':\ + \ {\n 'image_uri': 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/xgboost-training:20231002_0125',\n\ + \ 'args': [\n f'--job_dir={formatted_job_dir}',\n\ + \ f'--target_column={target_column}',\n f'--objective={objective}',\n\ + \ f'--training_data_path={get_gcs_path(materialized_train_split)}',\n\ + \ f'--validation_data_path={get_gcs_path(materialized_eval_split)}',\n\ + \ f'--transform_output_path={get_gcs_path(transform_output)}',\n\ + \ f'--training_schema_path={get_gcs_path(training_schema_uri)}',\n\ + \ f'--baseline_path={get_gcs_path(instance_baseline)}',\n \ + \ f'--eval_metric={eval_metric}',\n f'--num_boost_round={num_boost_round}',\n\ + \ f'--base_score={base_score}',\n f'--disable_default_eval_metric={disable_default_eval_metric}',\n\ + \ f'--seed={seed}',\n f'--seed_per_iteration={seed_per_iteration}',\n\ + \ f'--booster={booster}',\n f'--eta={eta}',\n\ + \ f'--gamma={gamma}',\n f'--max_depth={max_depth}',\n\ + \ f'--min_child_weight={min_child_weight}',\n \ + \ f'--max_delta_step={max_delta_step}',\n f'--subsample={subsample}',\n\ + \ f'--colsample_bytree={colsample_bytree}',\n \ + \ f'--colsample_bylevel={colsample_bylevel}',\n f'--colsample_bynode={colsample_bynode}',\n\ + \ f'--lambda={reg_lambda}',\n f'--alpha={reg_alpha}',\n\ + \ f'--tree_method={tree_method}',\n f'--scale_pos_weight={scale_pos_weight}',\n\ + \ f'--refresh_leaf={refresh_leaf}',\n f'--process_type={process_type}',\n\ + \ f'--grow_policy={grow_policy}',\n f'--sampling_method={sampling_method}',\n\ + \ f'--sample_type={sample_type}',\n f'--normalize_type={normalize_type}',\n\ + \ f'--rate_drop={rate_drop}',\n f'--one_drop={one_drop}',\n\ + \ f'--skip_drop={skip_drop}',\n f'--num_parallel_tree={num_parallel_tree}',\n\ + \ f'--feature_selector={feature_selector}',\n \ + \ f'--top_k={top_k}',\n f'--max_leaves={max_leaves}',\n \ + \ f'--max_bin={max_bin}',\n f'--tweedie_variance_power={tweedie_variance_power}',\n\ + \ f'--huber_slope={huber_slope}',\n f'--prediction_docker_uri={prediction_docker_uri}',\n\ + \ '--executor_input={{$.json_escape[1]}}',\n ],\n\ + \ },\n }\n\n # Add optional arguments if set\n if weight_column:\n\ + \ master_worker_pool_spec['container_spec']['args'].append(\n \ + \ f'--weight_column={weight_column}'\n )\n if early_stopping_rounds\ + \ >= 0:\n master_worker_pool_spec['container_spec']['args'].append(\n\ + \ f'--early_stopping_rounds={early_stopping_rounds}'\n )\n if\ + \ updater:\n master_worker_pool_spec['container_spec']['args'].append(\n\ + \ f'--updater={updater}'\n )\n if monotone_constraints:\n \ + \ master_worker_pool_spec['container_spec']['args'].append(\n f'--monotone_constraints={monotone_constraints}'\n\ + \ )\n if interaction_constraints:\n master_worker_pool_spec['container_spec']['args'].append(\n\ + \ f'--interaction_constraints={interaction_constraints}'\n )\n\ + \ if max_cat_to_onehot >= 0:\n master_worker_pool_spec['container_spec']['args'].append(\n\ + \ f'--max_cat_to_onehot={max_cat_to_onehot}'\n )\n\n # Add accelerator_type\ + \ and accelerator_count if set.\n if accelerator_type:\n if accelerator_count\ + \ <= 0:\n raise ValueError(\n 'Accelerator count must be greator\ + \ than 0 when type is specified.'\n )\n master_worker_pool_spec['machine_spec'][\n\ + \ 'accelerator_type'\n ] = accelerator_type\n master_worker_pool_spec['machine_spec'][\n\ + \ 'accelerator_count'\n ] = accelerator_count\n\n worker_pool_specs_lst\ + \ = [master_worker_pool_spec]\n\n # Add an additional worker pool spec\ + \ for distributed training.\n if total_replica_count > 1:\n additional_replica\ + \ = total_replica_count - 1\n additional_worker_spec = copy.deepcopy(master_worker_pool_spec)\n\ + \ additional_worker_spec['replica_count'] = additional_replica\n worker_pool_specs_lst.append(additional_worker_spec)\n\ + \n # Build unmanaged_container_model\n model_dir = os.path.join(formatted_job_dir,\ + \ 'model')\n unmanaged_container_model.metadata['containerSpec'] = {\n\ + \ 'imageUri': prediction_docker_uri,\n 'healthRoute': '/health',\n\ + \ 'predictRoute': '/predict',\n }\n unmanaged_container_model.metadata['predictSchemata']\ + \ = {\n 'instanceSchemaUri': os.path.join(model_dir, 'instance.yaml'),\n\ + \ 'predictionSchemaUri': os.path.join(model_dir, 'prediction_schema.yaml'),\n\ + \ }\n unmanaged_container_model.uri = model_dir\n\n return collections.namedtuple('Outputs',\ + \ ['worker_pool_specs'])(\n worker_pool_specs_lst\n )\n\n" + image: python:3.7 + exec-get-prediction-type-for-xgboost: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _get_prediction_type_for_xgboost + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _get_prediction_type_for_xgboost(objective: str) -> str:\n \"\ + \"\"Returns prediction_type given XGBoost training objective..\n\n Args:\n\ + \ objective: The XGBoost training objective\n\n Returns:\n A string.\ + \ One of 'regression' or 'classification'\n \"\"\"\n if objective.startswith('binary')\ + \ or objective.startswith('multi'):\n return 'classification'\n elif\ + \ objective.startswith('reg'):\n return 'regression'\n else:\n raise\ + \ ValueError(\n f'Unsupported XGBoost training objective: {objective}.\ + \ Must be one of'\n ' [reg:squarederror, reg:squaredlogerror, reg:logistic,\ + \ reg:gamma,'\n ' reg:tweedie, reg:pseudohubererror, binary:logistic,'\n\ + \ ' multi:softprob].'\n )\n\n" + image: python:3.7 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"pipeline_job\": + \"", "projects/{{$.inputs.parameters[''project'']}}/locations/{{$.inputs.parameters[''location'']}}/pipelineJobs/{{$.pipeline_job_uuid}}", + "\"", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.model.upload_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-split-materialized-data: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _split_materialized_data + command: + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _split_materialized_data(\n materialized_data: Input[Dataset],\n\ + \ materialized_train_split: OutputPath('MaterializedSplit'),\n materialized_eval_split:\ + \ OutputPath('MaterializedSplit'),\n materialized_test_split: OutputPath('MaterializedSplit')):\n\ + \ \"\"\"Splits materialized_data into materialized_data test, train, and\ + \ eval splits.\n\n Necessary adapter between FTE pipeline and trainer.\n\ + \n Args:\n materialized_data: materialized_data dataset output by FTE.\n\ + \ materialized_train_split: Path patern to materialized_train_split.\n\ + \ materialized_eval_split: Path patern to materialized_eval_split.\n\ + \ materialized_test_split: Path patern to materialized_test_split.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n artifact_path\ + \ = f.read()\n\n # needed to import tf because this is a path in gs://\n\ + \ with tf.io.gfile.GFile(artifact_path, 'r') as f:\n materialized_data_json\ + \ = json.load(f)\n\n if 'tf_record_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['tf_record_data_source'][\n\ + \ 'file_patterns']\n elif 'avro_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['avro_data_source'][\n \ + \ 'file_patterns']\n elif 'parquet_data_source' in materialized_data_json:\n\ + \ file_patterns = materialized_data_json['parquet_data_source'][\n \ + \ 'file_patterns']\n else:\n raise ValueError(f'Unsupported training\ + \ data source: {materialized_data_json}')\n\n # we map indices to file\ + \ patterns based on the ordering of insertion order\n # in our transform_data\ + \ (see above in _generate_analyze_and_transform_data)\n with tf.io.gfile.GFile(materialized_train_split,\ + \ 'w') as f:\n f.write(file_patterns[0])\n\n with tf.io.gfile.GFile(materialized_eval_split,\ + \ 'w') as f:\n f.write(file_patterns[1])\n\n with tf.io.gfile.GFile(materialized_test_split,\ + \ 'w') as f:\n f.write(file_patterns[2])\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + exec-training-configurator-and-validator: + container: + args: + - training_configurator_and_validator + - '{"Concat": ["--instance_schema_path=", "{{$.inputs.artifacts[''instance_schema''].uri}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.inputs.artifacts[''training_schema''].uri}}"]}' + - '{"Concat": ["--dataset_stats_path=", "{{$.inputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--split_example_counts=", "{{$.inputs.parameters[''split_example_counts'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"Concat": ["--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}"]}' + - '{"Concat": ["--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}"]}' + - '{"Concat": ["--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}"]}' + - '{"Concat": ["--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}"]}' + - '{"Concat": ["--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}"]}' + - '{"Concat": ["--run_evaluation=", "{{$.inputs.parameters[''run_evaluation'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--enable_probabilistic_inference=", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}"]}' + - '{"IfPresent": {"InputName": "time_series_identifier_column", "Then": {"Concat": + ["--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--time_series_identifier_columns=", "{{$.inputs.parameters[''time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--time_column=", "{{$.inputs.parameters[''time_column'']}}"]}' + - '{"Concat": ["--time_series_attribute_columns=", "{{$.inputs.parameters[''time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--available_at_forecast_columns=", "{{$.inputs.parameters[''available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--unavailable_at_forecast_columns=", "{{$.inputs.parameters[''unavailable_at_forecast_columns'']}}"]}' + - '{"IfPresent": {"InputName": "quantiles", "Then": {"Concat": ["--quantiles=", + "{{$.inputs.parameters[''quantiles'']}}"]}}}' + - '{"Concat": ["--context_window=", "{{$.inputs.parameters[''context_window'']}}"]}' + - '{"Concat": ["--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_model_type=", "{{$.inputs.parameters[''forecasting_model_type'']}}"]}' + - '{"Concat": ["--forecasting_transformations=", "{{$.inputs.parameters[''forecasting_transformations'']}}"]}' + - '{"IfPresent": {"InputName": "stage_1_deadline_hours", "Then": {"Concat": + ["--stage_1_deadline_hours=", "{{$.inputs.parameters[''stage_1_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "stage_2_deadline_hours", "Then": {"Concat": + ["--stage_2_deadline_hours=", "{{$.inputs.parameters[''stage_2_deadline_hours'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + exec-xgboost-trainer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"xgboost-trainer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": ", "{{$.inputs.parameters[''worker_pool_specs'']}}", + "}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 +pipelineInfo: + description: The XGBoost training pipeline. + name: automl-tabular-xgboost-trainer +root: + dag: + outputs: + artifacts: + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--base_score: + componentInputParameter: base_score + pipelinechannel--bigquery_staging_full_dataset_id: + componentInputParameter: bigquery_staging_full_dataset_id + pipelinechannel--booster: + componentInputParameter: booster + pipelinechannel--colsample_bylevel: + componentInputParameter: colsample_bylevel + pipelinechannel--colsample_bynode: + componentInputParameter: colsample_bynode + pipelinechannel--colsample_bytree: + componentInputParameter: colsample_bytree + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--dataset_level_custom_transformation_definitions: + componentInputParameter: dataset_level_custom_transformation_definitions + pipelinechannel--dataset_level_transformations: + componentInputParameter: dataset_level_transformations + pipelinechannel--disable_default_eval_metric: + componentInputParameter: disable_default_eval_metric + pipelinechannel--early_stopping_rounds: + componentInputParameter: early_stopping_rounds + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--eta: + componentInputParameter: eta + pipelinechannel--eval_metric: + componentInputParameter: eval_metric + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--feature_selection_algorithm: + componentInputParameter: feature_selection_algorithm + pipelinechannel--feature_selector: + componentInputParameter: feature_selector + pipelinechannel--gamma: + componentInputParameter: gamma + pipelinechannel--grow_policy: + componentInputParameter: grow_policy + pipelinechannel--huber_slope: + componentInputParameter: huber_slope + pipelinechannel--interaction_constraints: + componentInputParameter: interaction_constraints + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--max_bin: + componentInputParameter: max_bin + pipelinechannel--max_cat_to_onehot: + componentInputParameter: max_cat_to_onehot + pipelinechannel--max_delta_step: + componentInputParameter: max_delta_step + pipelinechannel--max_depth: + componentInputParameter: max_depth + pipelinechannel--max_leaves: + componentInputParameter: max_leaves + pipelinechannel--max_selected_features: + componentInputParameter: max_selected_features + pipelinechannel--min_child_weight: + componentInputParameter: min_child_weight + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--monotone_constraints: + componentInputParameter: monotone_constraints + pipelinechannel--normalize_type: + componentInputParameter: normalize_type + pipelinechannel--num_boost_round: + componentInputParameter: num_boost_round + pipelinechannel--num_parallel_tree: + componentInputParameter: num_parallel_tree + pipelinechannel--objective: + componentInputParameter: objective + pipelinechannel--one_drop: + componentInputParameter: one_drop + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--process_type: + componentInputParameter: process_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--rate_drop: + componentInputParameter: rate_drop + pipelinechannel--refresh_leaf: + componentInputParameter: refresh_leaf + pipelinechannel--reg_alpha: + componentInputParameter: reg_alpha + pipelinechannel--reg_lambda: + componentInputParameter: reg_lambda + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--run_feature_selection: + componentInputParameter: run_feature_selection + pipelinechannel--sample_type: + componentInputParameter: sample_type + pipelinechannel--sampling_method: + componentInputParameter: sampling_method + pipelinechannel--scale_pos_weight: + componentInputParameter: scale_pos_weight + pipelinechannel--seed: + componentInputParameter: seed + pipelinechannel--seed_per_iteration: + componentInputParameter: seed_per_iteration + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--skip_drop: + componentInputParameter: skip_drop + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--subsample: + componentInputParameter: subsample + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--tf_auto_transform_features: + componentInputParameter: tf_auto_transform_features + pipelinechannel--tf_custom_transformation_definitions: + componentInputParameter: tf_custom_transformation_definitions + pipelinechannel--tf_transformations_path: + componentInputParameter: tf_transformations_path + pipelinechannel--top_k: + componentInputParameter: top_k + pipelinechannel--training_accelerator_count: + componentInputParameter: training_accelerator_count + pipelinechannel--training_accelerator_type: + componentInputParameter: training_accelerator_type + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--training_machine_type: + componentInputParameter: training_machine_type + pipelinechannel--training_total_replica_count: + componentInputParameter: training_total_replica_count + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--tree_method: + componentInputParameter: tree_method + pipelinechannel--tweedie_variance_power: + componentInputParameter: tweedie_variance_power + pipelinechannel--updater: + componentInputParameter: updater + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Parent model if this model is uploaded as a version. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + base_score: + defaultValue: 0.5 + description: The initial prediction score of all instances, global bias. + isOptional: true + parameterType: NUMBER_DOUBLE + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'The BigQuery staging full dataset id for + + storing intermediate tables.' + isOptional: true + parameterType: STRING + booster: + defaultValue: gbtree + description: 'Which booster to use, can be gbtree, gblinear or dart. gbtree + and + + dart use tree based model while gblinear uses linear function.' + isOptional: true + parameterType: STRING + colsample_bylevel: + defaultValue: 1.0 + description: Subsample ratio of columns for each split, in each level. + isOptional: true + parameterType: NUMBER_DOUBLE + colsample_bynode: + defaultValue: 1.0 + description: Subsample ratio of columns for each node (split). + isOptional: true + parameterType: NUMBER_DOUBLE + colsample_bytree: + defaultValue: 1.0 + description: Subsample ratio of columns when constructing each tree. + isOptional: true + parameterType: NUMBER_DOUBLE + data_source_bigquery_table_path: + defaultValue: '' + description: The BigQuery data source. + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: The CSV data source. + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + description: 'Dataset-level custom + + transformation definitions in string format.' + isOptional: true + parameterType: LIST + dataset_level_transformations: + description: 'Dataset-level transformation configuration in + + string format.' + isOptional: true + parameterType: LIST + disable_default_eval_metric: + defaultValue: 0.0 + description: 'Flag to disable default metric. Set to >0 to + + disable. Default to 0.' + isOptional: true + parameterType: NUMBER_INTEGER + early_stopping_rounds: + defaultValue: -1.0 + description: 'Activates early stopping. Validation error needs to + + decrease at least every early_stopping_rounds round(s) to continue + + training.' + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + eta: + defaultValue: 0.3 + description: Learning rate. + isOptional: true + parameterType: NUMBER_DOUBLE + eval_metric: + defaultValue: '' + description: 'Evaluation metrics for validation data represented as a + + comma-separated string.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + feature_selection_algorithm: + defaultValue: AMI + description: Feature selection algorithm. + isOptional: true + parameterType: STRING + feature_selector: + defaultValue: cyclic + description: '[linear booster only] Feature selection and ordering + + method.' + isOptional: true + parameterType: STRING + gamma: + defaultValue: 0.0 + description: 'Minimum loss reduction required to make a further partition + on a leaf + + node of the tree.' + isOptional: true + parameterType: NUMBER_DOUBLE + grow_policy: + defaultValue: depthwise + description: 'Controls a way new nodes are added to the tree. Only supported + + if tree_method is hist. Choices:["depthwise", "lossguide"]' + isOptional: true + parameterType: STRING + huber_slope: + defaultValue: 1.0 + description: 'A parameter used for Pseudo-Huber loss to define the delta + + term.' + isOptional: true + parameterType: NUMBER_DOUBLE + interaction_constraints: + defaultValue: '' + description: 'Constraints for interaction representing permitted + + interactions.' + isOptional: true + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + max_bin: + defaultValue: 256.0 + description: Maximum number of discrete bins to bucket continuous features. + isOptional: true + parameterType: NUMBER_INTEGER + max_cat_to_onehot: + defaultValue: -1.0 + description: 'A threshold for deciding whether XGBoost should use + + one-hot encoding based split for categorical data.' + isOptional: true + parameterType: NUMBER_INTEGER + max_delta_step: + defaultValue: 0.0 + description: 'Maximum delta step we allow each tree''s weight estimation to + + be.' + isOptional: true + parameterType: NUMBER_DOUBLE + max_depth: + defaultValue: 6.0 + description: Maximum depth of a tree. + isOptional: true + parameterType: NUMBER_INTEGER + max_leaves: + defaultValue: 0.0 + description: Maximum number of nodes to be added. + isOptional: true + parameterType: NUMBER_INTEGER + max_selected_features: + defaultValue: -1.0 + description: Maximum number of features to select. + isOptional: true + parameterType: NUMBER_INTEGER + min_child_weight: + defaultValue: 1.0 + description: Minimum sum of instance weight(hessian) needed in a child. + isOptional: true + parameterType: NUMBER_DOUBLE + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model. + isOptional: true + parameterType: STRING + monotone_constraints: + defaultValue: '' + description: Constraint of variable monotonicity. + isOptional: true + parameterType: STRING + normalize_type: + defaultValue: tree + description: '[dart booster only] Type of normalization algorithm, + + Choices:["tree", "forest"]' + isOptional: true + parameterType: STRING + num_boost_round: + defaultValue: 10.0 + description: Number of boosting iterations. + isOptional: true + parameterType: NUMBER_INTEGER + num_parallel_tree: + defaultValue: 1.0 + description: 'Number of parallel trees constructed during each + + iteration. This option is used to support boosted random forest.' + isOptional: true + parameterType: NUMBER_INTEGER + objective: + description: 'Specifies the learning task and the learning objective. Must + be + + one of [reg:squarederror, reg:squaredlogerror, + + reg:logistic, reg:gamma, reg:tweedie, reg:pseudohubererror, + + binary:logistic, multi:softprob].' + parameterType: STRING + one_drop: + defaultValue: 0.0 + description: '[dart booster only] When this flag is enabled, at least one + tree + + is always dropped during the dropout (allows Binomial-plus-one or + + epsilon-dropout from the original DART paper).' + isOptional: true + parameterType: NUMBER_INTEGER + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + process_type: + defaultValue: default + description: 'A type of boosting process to run. Choices:["default", + + "update"]' + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + rate_drop: + defaultValue: 0.0 + description: '[dart booster only] Dropout rate.''' + isOptional: true + parameterType: NUMBER_DOUBLE + refresh_leaf: + defaultValue: 1.0 + description: 'Refresh updater plugin. Update tree leaf and nodes''s stats + if + + True. When it is False, only node stats are updated.' + isOptional: true + parameterType: NUMBER_INTEGER + reg_alpha: + defaultValue: 0.0 + description: L1 regularization term on weights. + isOptional: true + parameterType: NUMBER_DOUBLE + reg_lambda: + defaultValue: 1.0 + description: L2 regularization term on weights. + isOptional: true + parameterType: NUMBER_DOUBLE + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_evaluation: + defaultValue: true + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: Whether to enable feature selection. + isOptional: true + parameterType: BOOLEAN + sample_type: + defaultValue: uniform + description: '[dart booster only] Type of sampling algorithm. + + Choices:["uniform", "weighted"]' + isOptional: true + parameterType: STRING + sampling_method: + defaultValue: uniform + description: The method to use to sample the training instances. + isOptional: true + parameterType: STRING + scale_pos_weight: + defaultValue: 1.0 + description: Control the balance of positive and negative weights. + isOptional: true + parameterType: NUMBER_DOUBLE + seed: + defaultValue: 0.0 + description: Random seed. + isOptional: true + parameterType: NUMBER_INTEGER + seed_per_iteration: + defaultValue: false + description: Seed PRNG determnisticly via iterator number. + isOptional: true + parameterType: BOOLEAN + skip_drop: + defaultValue: 0.0 + description: '[dart booster only] Probability of skipping the dropout procedure + + during a boosting iteration.' + isOptional: true + parameterType: NUMBER_DOUBLE + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + subsample: + defaultValue: 1.0 + description: Subsample ratio of the training instance. + isOptional: true + parameterType: NUMBER_DOUBLE + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: Test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + description: 'List of auto transform features in the + + comma-separated string format.' + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + description: 'TF custom transformation definitions + + in string format.' + isOptional: true + parameterType: LIST + tf_transformations_path: + defaultValue: '' + description: Path to TF transformation configuration. + isOptional: true + parameterType: STRING + top_k: + defaultValue: 0.0 + description: 'The number of top features to select in greedy and thrifty feature + + selector. The value of 0 means using all the features.' + isOptional: true + parameterType: NUMBER_INTEGER + training_accelerator_count: + defaultValue: 0.0 + description: Accelerator count. + isOptional: true + parameterType: NUMBER_INTEGER + training_accelerator_type: + defaultValue: '' + description: Accelerator type. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + training_machine_type: + defaultValue: c2-standard-16 + description: Machine type. + isOptional: true + parameterType: STRING + training_total_replica_count: + defaultValue: 1.0 + description: Number of workers. + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + tree_method: + defaultValue: auto + description: 'The tree construction algorithm used in XGBoost. Choices: + + ["auto", "exact", "approx", "hist", "gpu_exact", "gpu_hist"].' + isOptional: true + parameterType: STRING + tweedie_variance_power: + defaultValue: 1.5 + description: 'Parameter that controls the variance of the Tweedie + + distribution.' + isOptional: true + parameterType: NUMBER_DOUBLE + updater: + defaultValue: '' + description: 'A comma separated string defining the sequence of tree updaters + to + + run.' + isOptional: true + parameterType: STRING + validation_fraction: + defaultValue: -1.0 + description: Validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/version.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/vision/__init__.py similarity index 76% rename from components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/version.py rename to components/google-cloud/google_cloud_pipeline_components/preview/automl/vision/__init__.py index 0880702fba..d854e1cd11 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/version.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/vision/__init__.py @@ -11,9 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Version constants for model evaluation components.""" +"""AutoML Vision components.""" -EVAL_VERSION = 'v0.9.2' +from google_cloud_pipeline_components.preview.automl.vision.data_converter import data_converter as DataConverterJobOp -EVAL_IMAGE_NAME = 'gcr.io/ml-pipeline/model-evaluation' -EVAL_IMAGE_TAG = f'{EVAL_IMAGE_NAME}:{EVAL_VERSION}' +__all__ = [ + 'DataConverterJobOp', +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/automl/vision/data_converter.py b/components/google-cloud/google_cloud_pipeline_components/preview/automl/vision/data_converter.py new file mode 100644 index 0000000000..6ccc98dd40 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/automl/vision/data_converter.py @@ -0,0 +1,152 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AutoML Vision training data converter.""" + +from typing import Optional + +from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders +from kfp import dsl + + +# pylint: disable=g-doc-args +@dsl.container_component +def data_converter( + display_name: str, + input_file_path: str, + input_file_type: str, + objective: str, + output_dir: dsl.OutputPath(str), + gcp_resources: dsl.OutputPath(str), + location: str = 'us-central1', + timeout: str = '604800s', + service_account: Optional[str] = None, + machine_type: str = 'n1-highmem-4', + output_shape: Optional[str] = None, + split_ratio: Optional[str] = None, + num_shard: Optional[str] = None, + encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +): + # fmt: off + """Runs AutoML Vision data conversion. It will be launched as a Vertex AI [custom training job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) using the [CustomJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.customJobs) API. + + Args: + display_name: The name of the CustomJob. + input_file_path: Input file path. Please refer to different input formats in Vertex AI Documentation. For example, [image classification prepare data](https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data) page. + input_file_type: 'csv', 'jsonl', or 'coco_json'. Must be one of the input file types supported by the objective. + objective: One of 'icn', 'iod', 'isg', 'vcn', or 'var'. + location: Location for creating the custom training job. If not set, default to us-central1. + timeout: The maximum job running time. The default is 7 days. A duration in seconds with up to nine fractional digits, terminated by 's', for example: "3.5s". + service_account: Sets the default service account for workload run-as account. The [service account](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) running the pipeline submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code [Service Agent ](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. + machine_type: [Machine type](https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types) for the CustomJob. If conversion failed, consider using a machine type with more RAM or splitting dataset into smaller pieces. + output_shape: Output shape (height,width) for images. + split_ratio: Proportion of data to split into train/validation/test, separated by comma. + num_shard: Number of train/validation/test shards, separated by comma. + encryption_spec_key_name: Customer-managed encryption key options for the CustomJob. If this is set, then all resources created by the CustomJob will be encrypted with the provided encryption key. + project: Project to create the custom training job in. Defaults to the project in which the PipelineJob is run. + Returns: + output_dir: Cloud Storage directory storing converted data and pipeline information. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the CustomJob. + """ + # fmt: on + return dsl.ContainerSpec( + image=_image.GCPC_IMAGE_TAG, + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--payload', + dsl.ConcatPlaceholder([ + '{', + '"display_name": "', + display_name, + '",', + '"job_spec": {', + '"worker_pool_specs": [{', + '"machine_spec": {', + '"machine_type": "', + machine_type, + '"},', + '"replica_count": 1,', + '"container_spec": {', + ( + '"image_uri":' + ' "us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/data-converter",' + ), + '"args": [', + '"--enable_input_validation","true",', + '"--input_file_path", "', + input_file_path, + '",', + '"--input_file_type", "', + input_file_type, + '",', + '"--objective", "', + objective, + '",', + '"--output_dir", "', + output_dir, + '"', + dsl.IfPresentPlaceholder( + input_name='output_shape', + then=dsl.ConcatPlaceholder( + [',"--output_shape","', output_shape, '"'] + ), + ), + dsl.IfPresentPlaceholder( + input_name='split_ratio', + then=dsl.ConcatPlaceholder( + [',"--split_ratio","', split_ratio, '"'] + ), + ), + dsl.IfPresentPlaceholder( + input_name='num_shard', + then=dsl.ConcatPlaceholder( + [',"--num_shard","', num_shard, '"'] + ), + ), + ']}}],', + '"scheduling": {', + '"timeout": "', + timeout, + '"', + '},', + dsl.IfPresentPlaceholder( + input_name='service_account', + then=dsl.ConcatPlaceholder( + ['"service_account": "', service_account, '",'] + ), + ), + '"enable_web_access": false', + '},', + '"encryption_spec": {', + '"kms_key_name": "', + encryption_spec_key_name, + '"', + '}}', + ]), + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/__init__.py new file mode 100644 index 0000000000..ac8c9aeb78 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# fmt: off +"""Run KFP components as [Vertex AI Custom Training Jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) with +customized worker and cloud configurations. +""" +# fmt: on + +from google_cloud_pipeline_components.preview.custom_job.component import custom_training_job as CustomTrainingJobOp +from google_cloud_pipeline_components.preview.custom_job.utils import create_custom_training_job_from_component +from google_cloud_pipeline_components.preview.custom_job.utils import create_custom_training_job_op_from_component + +__all__ = [ + 'CustomTrainingJobOp', + 'create_custom_training_job_op_from_component', + 'create_custom_training_job_from_component', +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/component.py b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/component.py new file mode 100644 index 0000000000..b155e39130 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/component.py @@ -0,0 +1,108 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List + +from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components import utils +from kfp import dsl + + +# keep consistent with create_custom_training_job_from_component +@dsl.container_component +def custom_training_job( + display_name: str, + gcp_resources: dsl.OutputPath(str), + location: str = 'us-central1', + worker_pool_specs: List[Dict[str, str]] = [], + timeout: str = '604800s', + restart_job_on_worker_restart: bool = False, + service_account: str = '', + tensorboard: str = '', + enable_web_access: bool = False, + network: str = '', + reserved_ip_ranges: List[str] = [], + base_output_directory: str = '', + labels: Dict[str, str] = {}, + encryption_spec_key_name: str = '', + persistent_resource_id: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +): + # fmt: off + """Launch a Vertex AI [custom training job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) using the [CustomJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.customJobs) API. See [Create custom training jobs ](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) for more information. + + Args: + location: Location for creating the custom training job. If not set, default to us-central1. + display_name: The name of the CustomJob. + worker_pool_specs: Serialized json spec of the worker pools including machine type and Docker image. All worker pools except the first one are optional and can be skipped by providing an empty value. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec#WorkerPoolSpec). + timeout: The maximum job running time. The default is 7 days. A duration in seconds with up to nine fractional digits, terminated by 's', for example: "3.5s". + restart_job_on_worker_restart: Restarts the entire CustomJob if a worker gets restarted. This feature can be used by distributed training jobs that are not resilient to workers leaving and joining a job. + service_account: Sets the default service account for workload run-as account. The [service account ](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) running the pipeline submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code [Service Agent ](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. + tensorboard: The name of a Vertex AI TensorBoard resource to which this CustomJob will upload TensorBoard logs. + enable_web_access: Whether you want Vertex AI to enable [interactive shell access ](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) to training containers. If `True`, you can access interactive shells at the URIs given by [CustomJob.web_access_uris][]. + network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name. Private services access must already be configured for the network. If left unspecified, the job is not peered with any network. + reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC network that can be used for this job. If set, we will deploy the job within the provided IP ranges. Otherwise, the job will be deployed to any IP ranges under the provided VPC network. + base_output_directory: The Cloud Storage location to store the output of this CustomJob or HyperparameterTuningJob. See [more information ](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/GcsDestination). + labels: The labels with user-defined metadata to organize the CustomJob. See [more information](https://goo.gl/xmQnxf). + encryption_spec_key_name: Customer-managed encryption key options for the CustomJob. If this is set, then all resources created by the CustomJob will be encrypted with the provided encryption key. + persistent_resource_id: The ID of the PersistentResource in the same Project and Location which to run. If this is specified, the job will be run on existing machines held by the PersistentResource instead of on-demand short-live machines. The network and CMEK configs on the job should be consistent with those on the PersistentResource, otherwise, the job will be rejected. (This is a Preview feature not yet recommended for production workloads.) + project: Project to create the custom training job in. Defaults to the project in which the PipelineJob is run. + Returns: + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the CustomJob. + """ + # fmt: on + return dsl.ContainerSpec( + image=_image.GCPC_IMAGE_TAG, + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.preview.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--payload', + utils.container_component_dumps({ + 'display_name': display_name, + 'job_spec': { + 'worker_pool_specs': worker_pool_specs, + 'scheduling': { + 'timeout': timeout, + 'restart_job_on_worker_restart': ( + restart_job_on_worker_restart + ), + }, + 'service_account': service_account, + 'tensorboard': tensorboard, + 'enable_web_access': enable_web_access, + 'network': network, + 'reserved_ip_ranges': reserved_ip_ranges, + 'base_output_directory': { + 'output_uri_prefix': base_output_directory + }, + 'persistent_resource_id': persistent_resource_id, + }, + 'labels': labels, + 'encryption_spec': {'kms_key_name': encryption_spec_key_name}, + }), + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/utils.py b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/utils.py new file mode 100644 index 0000000000..93bc3221a3 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/custom_job/utils.py @@ -0,0 +1,282 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for supporting Google Vertex AI Custom Training Job Op.""" + +import copy +import textwrap +from typing import Callable, Dict, List, Optional +import warnings + +from google_cloud_pipeline_components.preview.custom_job import component +from kfp import components +import yaml + +from google.protobuf import json_format + + +def _replace_executor_placeholder( + container_input: List[str], +) -> List[str]: + """Replace executor placeholder in container command or args. + + Args: + container_input: Container command or args. + + Returns: container_input with executor placeholder replaced. + """ + # Executor replacement is used as executor content needs to be jsonified before + # injection into the payload, since payload is already a JSON serialized string. + EXECUTOR_INPUT_PLACEHOLDER = '{{$}}' + JSON_ESCAPED_EXECUTOR_INPUT_PLACEHOLDER = '{{$.json_escape[1]}}' + return [ + JSON_ESCAPED_EXECUTOR_INPUT_PLACEHOLDER + if cmd_part == EXECUTOR_INPUT_PLACEHOLDER + else cmd_part + for cmd_part in container_input + ] + + +# keep identical to CustomTrainingJobOp +def create_custom_training_job_from_component( + component_spec: Callable, + display_name: str = '', + replica_count: int = 1, + machine_type: str = 'n1-standard-4', + accelerator_type: str = '', + accelerator_count: int = 1, + boot_disk_type: str = 'pd-ssd', + boot_disk_size_gb: int = 100, + timeout: str = '604800s', + restart_job_on_worker_restart: bool = False, + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', + tensorboard: str = '', + enable_web_access: bool = False, + reserved_ip_ranges: Optional[List[str]] = None, + nfs_mounts: Optional[List[Dict[str, str]]] = None, + base_output_directory: str = '', + labels: Optional[Dict[str, str]] = None, + persistent_resource_id: str = '', +) -> Callable: + # fmt: off + """Convert a KFP component into Vertex AI [custom training job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) using the [CustomJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.customJobs) API. + + This utility converts a [KFP component](https://www.kubeflow.org/docs/components/pipelines/v2/components/) provided to `component_spec` into `CustomTrainingJobOp` component. Your components inputs, outputs, and logic are carried over, with additional [CustomJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec) parameters exposed. Note that this utility constructs a ClusterSpec where the master and all the workers use the same spec, meaning all disk/machine spec related parameters will apply to all replicas. This is suitable for uses cases such as executing a training component over multiple replicas with [MultiWorkerMirroredStrategy](https://www.tensorflow.org/api_docs/python/tf/distribute/MultiWorkerMirroredStrategy) or [MirroredStrategy](https://www.tensorflow.org/api_docs/python/tf/distribute/MirroredStrategy). See [Create custom training jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) for more information. + + Args: + component_spec: A KFP component. + display_name: The name of the CustomJob. If not provided the component's name will be used instead. + replica_count: The count of instances in the cluster. One replica always counts towards the master in worker_pool_spec[0] and the remaining replicas will be allocated in worker_pool_spec[1]. See [more information.](https://cloud.google.com/vertex-ai/docs/training/distributed-training#configure_a_distributed_training_job) + machine_type: The type of the machine to run the CustomJob. The default value is "n1-standard-4". See [more information](https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types). + accelerator_type: The type of accelerator(s) that may be attached to the machine per `accelerator_count`. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec#acceleratortype). + accelerator_count: The number of accelerators to attach to the machine. Defaults to 1 if `accelerator_type` is set. + boot_disk_type: Type of the boot disk (default is "pd-ssd"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive). boot_disk_type is set as a static value and cannot be changed as a pipeline parameter. + boot_disk_size_gb: Size in GB of the boot disk (default is 100GB). `boot_disk_size_gb` is set as a static value and cannot be changed as a pipeline parameter. + timeout: The maximum job running time. The default is 7 days. A duration in seconds with up to nine fractional digits, terminated by 's', for example: "3.5s". + restart_job_on_worker_restart: Restarts the entire CustomJob if a worker gets restarted. This feature can be used by distributed training jobs that are not resilient to workers leaving and joining a job. + service_account: Sets the default service account for workload run-as account. The [service account](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) running the pipeline submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code [Service Agent](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name. Private services access must already be configured for the network. If left unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the CustomJob. If this is set, then all resources created by the CustomJob will be encrypted with the provided encryption key. + tensorboard: The name of a Vertex AI TensorBoard resource to which this CustomJob will upload TensorBoard logs. + enable_web_access: Whether you want Vertex AI to enable [interactive shell access](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) to training containers. If `True`, you can access interactive shells at the URIs given by [CustomJob.web_access_uris][]. + reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC network that can be used for this job. If set, we will deploy the job within the provided IP ranges. Otherwise, the job will be deployed to any IP ranges under the provided VPC network. + nfs_mounts: A list of [NfsMount](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec#NfsMount) resource specs in Json dict format. For more details about mounting NFS for CustomJob, see [Mount an NFS share for custom training](https://cloud.google.com/vertex-ai/docs/training/train-nfs-share). + base_output_directory: The Cloud Storage location to store the output of this CustomJob or HyperparameterTuningJob. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/GcsDestination). + labels: The labels with user-defined metadata to organize the CustomJob. See [more information](https://goo.gl/xmQnxf). + persistent_resource_id: The ID of the PersistentResource in the same Project and Location which to run. If this is specified, the job will be run on existing machines held by the PersistentResource instead of on-demand short-live machines. The network and CMEK configs on the job should be consistent with those on the PersistentResource, otherwise, the job will be rejected. (This is a Preview feature not yet recommended for production workloads.) + + Returns: + A KFP component with CustomJob specification applied. + """ + # fmt: on + # This function constructs a Custom Job component based on the input + # component, by performing a 3-way merge of the inputs/outputs of the + # input component, the Custom Job component and the arguments given to this + # function. + # + # It first retrieves the PipelineSpec (as a Python dict) for each of the two + # components (the input component and the Custom Job component). + # Note: The advantage of using the PipelineSpec here is that the + # placeholders are (mostly) serialized, so there is less processing + # needed (and avoids unnecessary dependency on KFP internals). + # + # The arguments to this function are first inserted into each input parameter + # of the Custom Job component as a default value (which will be used at + # runtime, unless when overridden by specifying the input). + # One particular input parameter that needs detailed construction is the + # worker_pool_spec, before being inserted into the Custom Job component. + # + # After inserting the arguments into the Custom Job input parameters as + # default values, the input/output parameters from the input component are + # then merged with the Custom Job input/output parameters. Preference is given + # to Custom Job input parameters to make sure they are not overridden (which + # follows the same logic as the original version). + # + # It is assumed that the Custom Job component itself has no input/output + # artifacts, so the artifacts from the input component needs no merging. + # (There is a unit test to make sure this is the case, otherwise merging of + # artifacts need to be done here.) + # + # Once the above is done, and the dict of the Custom Job is converted back + # into a KFP component (by first converting to YAML, then using + # load_component_from_text to load the YAML). + # After adding the appropriate description and the name, the new component + # is returned. + + cj_pipeline_spec = json_format.MessageToDict( + component.custom_training_job.pipeline_spec + ) + user_pipeline_spec = json_format.MessageToDict(component_spec.pipeline_spec) + + user_component_container = list( + user_pipeline_spec['deploymentSpec']['executors'].values() + )[0]['container'] + + worker_pool_spec = { + 'machine_spec': {'machine_type': machine_type}, + 'replica_count': 1, + 'container_spec': { + 'image_uri': user_component_container['image'], + 'command': _replace_executor_placeholder( + user_component_container.get('command', []) + ), + 'args': _replace_executor_placeholder( + user_component_container.get('args', []) + ), + }, + } + if accelerator_type: + worker_pool_spec['machine_spec']['accelerator_type'] = accelerator_type + worker_pool_spec['machine_spec']['accelerator_count'] = accelerator_count + if boot_disk_type: + worker_pool_spec['disk_spec'] = { + 'boot_disk_type': boot_disk_type, + 'boot_disk_size_gb': boot_disk_size_gb, + } + if nfs_mounts: + worker_pool_spec['nfs_mounts'] = nfs_mounts + + worker_pool_specs = [worker_pool_spec] + + if int(replica_count) > 1: + additional_worker_pool_spec = copy.deepcopy(worker_pool_spec) + additional_worker_pool_spec['replica_count'] = replica_count - 1 + worker_pool_specs.append(additional_worker_pool_spec) + + # get the component spec for both components + cj_component_spec_key = list(cj_pipeline_spec['components'].keys())[0] + cj_component_spec = cj_pipeline_spec['components'][cj_component_spec_key] + + user_component_spec_key = list(user_pipeline_spec['components'].keys())[0] + user_component_spec = user_pipeline_spec['components'][ + user_component_spec_key + ] + + # add custom job defaults based on user-provided args + custom_job_param_defaults = { + 'display_name': display_name or component_spec.component_spec.name, + 'worker_pool_specs': worker_pool_specs, + 'timeout': timeout, + 'restart_job_on_worker_restart': restart_job_on_worker_restart, + 'service_account': service_account, + 'tensorboard': tensorboard, + 'enable_web_access': enable_web_access, + 'network': network, + 'reserved_ip_ranges': reserved_ip_ranges or [], + 'base_output_directory': base_output_directory, + 'labels': labels or {}, + 'encryption_spec_key_name': encryption_spec_key_name, + 'persistent_resource_id': persistent_resource_id, + } + + for param_name, default_value in custom_job_param_defaults.items(): + cj_component_spec['inputDefinitions']['parameters'][param_name][ + 'defaultValue' + ] = default_value + + # merge parameters from user component into the customjob component + cj_component_spec['inputDefinitions']['parameters'].update( + user_component_spec.get('inputDefinitions', {}).get('parameters', {}) + ) + cj_component_spec['outputDefinitions']['parameters'].update( + user_component_spec.get('outputDefinitions', {}).get('parameters', {}) + ) + # use artifacts from user component + ## assign artifacts, not update, since customjob has no artifact outputs + cj_component_spec['inputDefinitions']['artifacts'] = user_component_spec.get( + 'inputDefinitions', {} + ).get('artifacts', {}) + cj_component_spec['outputDefinitions']['artifacts'] = user_component_spec.get( + 'outputDefinitions', {} + ).get('artifacts', {}) + + # copy the input definitions to the root, which will have an identical interface for a single-step pipeline + cj_pipeline_spec['root']['inputDefinitions'] = copy.deepcopy( + cj_component_spec['inputDefinitions'] + ) + cj_pipeline_spec['root']['outputDefinitions'] = copy.deepcopy( + cj_component_spec['outputDefinitions'] + ) + + # update the customjob task with the user inputs + cj_task_key = list(cj_pipeline_spec['root']['dag']['tasks'].keys())[0] + user_task_key = list(user_pipeline_spec['root']['dag']['tasks'].keys())[0] + + cj_pipeline_spec['root']['dag']['tasks'][cj_task_key]['inputs'].update( + user_pipeline_spec['root']['dag']['tasks'][user_task_key].get( + 'inputs', {} + ) + ) + + # reload the pipelinespec as a component using KFP + new_component = components.load_component_from_text( + yaml.safe_dump(cj_pipeline_spec) + ) + + # Copy the component name and description + # TODO(b/262360354): The inner .component_spec.name is needed here as that is + # the name that is retrieved by the FE for display. Can simply reference the + # outer .name once setter is implemented. + new_component.component_spec.name = component_spec.component_spec.name + + if component_spec.description: + component_description = textwrap.dedent(f""" + A CustomJob that wraps {component_spec.component_spec.name}. + + Original component description: + {component_spec.description} + + Custom Job wrapper description: + {component.custom_training_job.description} + """) + new_component.description = component_description + + return new_component + + +def create_custom_training_job_op_from_component(*args, **kwargs) -> Callable: + """Deprecated. + + Please use create_custom_training_job_from_component instead. + """ + + warnings.warn( + f'{create_custom_training_job_op_from_component.__name__!r} is' + ' deprecated. Please use' + f' {create_custom_training_job_from_component.__name__!r} instead.', + DeprecationWarning, + ) + + return create_custom_training_job_from_component(*args, **kwargs) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/dataflow/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/dataflow/__init__.py index 90f0263272..7a63f52c6c 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/dataflow/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/dataflow/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. """Dataflow preview components.""" -from google_cloud_pipeline_components.preview.dataflow.flex_template.component import dataflow_flex_template as DataflowFlexTemplateJobOp +from google_cloud_pipeline_components.v1.dataflow import DataflowFlexTemplateJobOp __all__ = [ 'DataflowFlexTemplateJobOp', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/llm/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/llm/__init__.py new file mode 100644 index 0000000000..e35f70ef04 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/llm/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Large-language model preview components.""" + +from google_cloud_pipeline_components.preview.llm.infer.component import infer_pipeline +from google_cloud_pipeline_components.preview.llm.rlhf.component import rlhf_pipeline + +__all__ = [ + 'infer_pipeline', + 'rlhf_pipeline', +] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/llm/infer/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/llm/infer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/llm/infer/component.py b/components/google-cloud/google_cloud_pipeline_components/preview/llm/infer/component.py new file mode 100644 index 0000000000..429916e434 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/llm/infer/component.py @@ -0,0 +1,121 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pipeline that performs bulk inference using a large-language model.""" + +from typing import NamedTuple, Optional + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.llm import bulk_inferrer +from google_cloud_pipeline_components._implementation.llm import env +from google_cloud_pipeline_components._implementation.llm import function_based +from google_cloud_pipeline_components._implementation.llm import private_text_importer +import kfp + +PipelineOutput = NamedTuple('Outputs', output_prediction_gcs_path=str) + + +@kfp.dsl.pipeline( + name='infer-eval-template', + description=( + 'Performs bulk inference on a dataset using a model checkpoint.' + ), +) +def infer_pipeline( + large_model_reference: str, + prompt_dataset: str, + model_checkpoint: Optional[str] = None, + prompt_sequence_length: int = 512, + target_sequence_length: int = 64, + sampling_strategy: str = 'greedy', + instruction: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, + location: str = _placeholders.LOCATION_PLACEHOLDER, +) -> PipelineOutput: + # fmt: off + """Uses a large-language model to perform bulk inference on a prompt dataset. + + Args: + large_model_reference: Name of the base model. Supported values are `text-bison@001`, `t5-small`, `t5-large`, `t5-xl` and `t5-xxl`. `text-bison@001` and `t5-small` are supported in `us-central1` and `europe-west4`. `t5-large`, `t5-xl` and `t5-xxl` are only supported in `europe-west4`. + model_checkpoint: Optional Cloud storage path to the model checkpoint. If not provided, the default checkpoint for the `large_model_reference` will be used. + prompt_dataset: Cloud storage path to an unlabled prompt dataset used for reinforcement learning. The dataset format is jsonl. Each example in the dataset must have an `input_text` field that contains the prompt. + prompt_sequence_length: Maximum tokenized sequence length for input text. Higher values increase memory overhead. This value should be at most 8192. Default value is 512. + target_sequence_length: Maximum tokenized sequence length for target text. Higher values increase memory overhead. This value should be at most 1024. Default value is 64. + sampling_strategy: This field specifies the sampling strategy. The valid options are 'greedy' and 'temperature_sampling'. + instruction: This field lets the model know what task it needs to perform. Base models have been trained over a large set of varied instructions. You can give a simple and intuitive description of the task and the model will follow it, e.g. "Classify this movie review as positive or negative" or "Translate this sentence to Danish". Do not specify this if your dataset already prepends the instruction to the inputs field. + project: Project used to run custom jobs. If not specified the project used to run the pipeline will be used. + location: Location used to run custom jobs. If not specified the location used to run the pipeline will be used. + + Returns: + Cloud storage path to output predictions. + """ + # fmt: on + prompt_column = 'input_text' + machine_spec = function_based.resolve_machine_spec( + location=location, + use_test_spec=env.get_use_test_machine_spec(), + ).set_display_name('Resolve Machine Spec') + reference_model_metadata = function_based.resolve_reference_model_metadata( + large_model_reference=large_model_reference, + reference_model_path=model_checkpoint, + ).set_display_name('Resolve Model Metadata') + + prompt_dataset_image_uri = function_based.resolve_private_image_uri( + image_name='text_importer', + ).set_display_name('Resolve Prompt Dataset Image URI') + prompt_dataset_importer = ( + private_text_importer.PrivateTextImporter( + project=project, + location=location, + input_text=prompt_dataset, + inputs_field_name=prompt_column, + targets_field_name='', # ignore targets_field_name + output_split_name=env.TRAIN_SPLIT, + large_model_reference=reference_model_metadata.outputs[ + 'large_model_reference' + ], + image_uri=prompt_dataset_image_uri.output, + instruction=instruction, + ) + .set_display_name('Import Prompt Dataset') + .set_caching_options(False) + ) + + bulk_inferrer_image_uri = function_based.resolve_private_image_uri( + image_name='infer', + accelerator_type=machine_spec.outputs['accelerator_type'], + accelerator_count=machine_spec.outputs['accelerator_count'], + ).set_display_name('Resolve Bulk Inferrer Image URI') + bulk_inference = bulk_inferrer.BulkInferrer( + project=project, + location=location, + input_model=reference_model_metadata.outputs['reference_model_path'], + input_dataset_path=prompt_dataset_importer.outputs['imported_data_path'], + dataset_split=env.TRAIN_SPLIT, + inputs_sequence_length=prompt_sequence_length, + targets_sequence_length=target_sequence_length, + large_model_reference=reference_model_metadata.outputs[ + 'large_model_reference' + ], + sampling_strategy=sampling_strategy, + accelerator_type=machine_spec.outputs['accelerator_type'], + accelerator_count=machine_spec.outputs['accelerator_count'], + machine_type=machine_spec.outputs['machine_type'], + image_uri=bulk_inferrer_image_uri.output, + ).set_display_name('Bulk Inferrer') + + return PipelineOutput( + output_prediction_gcs_path=bulk_inference.outputs[ + 'output_prediction_gcs_path' + ] + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/llm/rlhf/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/llm/rlhf/__init__.py new file mode 100644 index 0000000000..aa8704bef8 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/llm/rlhf/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/llm/rlhf/component.py b/components/google-cloud/google_cloud_pipeline_components/preview/llm/rlhf/component.py new file mode 100644 index 0000000000..d48e5b6a77 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/llm/rlhf/component.py @@ -0,0 +1,320 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defines a pipeline that performs reinforcement learning from human feedback.""" + +import json +from typing import NamedTuple, Optional + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.llm import deploy_llm_model +from google_cloud_pipeline_components._implementation.llm import env +from google_cloud_pipeline_components._implementation.llm import function_based +from google_cloud_pipeline_components._implementation.llm import private_text_comparison_importer +from google_cloud_pipeline_components._implementation.llm import private_text_importer +from google_cloud_pipeline_components._implementation.llm import reinforcer +from google_cloud_pipeline_components._implementation.llm import reward_model_trainer +from google_cloud_pipeline_components._implementation.llm import upload_llm_model +from google_cloud_pipeline_components._implementation.llm import upload_tensorboard_metrics +from google_cloud_pipeline_components.preview.llm.infer import component +import kfp + +PipelineOutput = NamedTuple( + 'Outputs', model_resource_name=str, endpoint_resource_name=str +) + + +@kfp.dsl.pipeline( + name='rlhf-train-template', + description='Performs reinforcement learning from human feedback.', +) +def rlhf_pipeline( + prompt_dataset: str, + preference_dataset: str, + large_model_reference: str, + model_display_name: Optional[str] = None, + prompt_sequence_length: int = 512, + target_sequence_length: int = 64, + reward_model_learning_rate_multiplier: float = 1.0, + reinforcement_learning_rate_multiplier: float = 1.0, + reward_model_train_steps: int = 1000, + reinforcement_learning_train_steps: int = 1000, + kl_coeff: float = 0.1, + instruction: Optional[str] = None, + deploy_model: bool = True, + eval_dataset: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, + location: str = _placeholders.LOCATION_PLACEHOLDER, + tensorboard_resource_id: Optional[str] = None, +) -> PipelineOutput: + # fmt: off + """Performs reinforcement learning from human feedback. + + Args: + prompt_dataset: Cloud storage path to an unlabled prompt dataset used for reinforcement learning. The dataset format is jsonl. Each example in the dataset must have an `input_text` field that contains the prompt. + preference_dataset: Cloud storage path to a human preference dataset used to train a reward model. The dataset format is jsonl. Each example in the dataset must contain the following fields: `input_text` that contains the prompt, `candidate_0` and `candidate_1` that contain candidate responses, `choice` that specifies the preferred candidate. + large_model_reference: Name of the base model. Supported values are `text-bison@001`, `t5-small`, `t5-large`, `t5-xl` and `t5-xxl`. `text-bison@001` and `t5-small` are supported in `us-central1` and `europe-west4`. `t5-large`, `t5-xl` and `t5-xxl` are only supported in `europe-west4`. + model_display_name: Name of the fine-tuned model shown in the Model Registry. If not provided, a default name will be created. + prompt_sequence_length: Maximum tokenized sequence length for input text. Higher values increase memory overhead. This value should be at most 8192. Default value is 512. + target_sequence_length: Maximum tokenized sequence length for target text. Higher values increase memory overhead. This value should be at most 1024. Default value is 64. + reward_model_learning_rate_multiplier: Constant used to adjust the base learning rate used when training a reward model. Multiply by a number > 1 to increase the magnitude of updates applied at each training step or multiply by a number < 1 to decrease the magnitude of updates. Default value is 1.0. + reinforcement_learning_rate_multiplier: Constant used to adjust the base learning rate used during reinforcement learning. Multiply by a number > 1 to increase the magnitude of updates applied at each training step or multiply by a number < 1 to decrease the magnitude of updates. Default value is 1.0. + reward_model_train_steps: Number of steps to use when training a reward model. Default value is 1000. + reinforcement_learning_train_steps: Number of reinforcement learning steps to perform when tuning a base model. Default value is 1000. + kl_coeff: Coefficient for KL penalty. This regularizes the policy model and penalizes if it diverges from its initial distribution. If set to 0, the reference language model is not loaded into memory. Default value is 0.1. + instruction: This field lets the model know what task it needs to perform. Base models have been trained over a large set of varied instructions. You can give a simple and intuitive description of the task and the model will follow it, e.g. "Classify this movie review as positive or negative" or "Translate this sentence to Danish". Do not specify this if your dataset already prepends the instruction to the inputs field. + deploy_model: Whether to deploy the model to an endpoint in `us-central1`. Default is True. + eval_dataset: Optional Cloud storage path to an evaluation dataset. If provided, inference will be performed on this dataset after training. The dataset format is jsonl. Each example in the dataset must contain a field `input_text` that contains the prompt. + project: Project used to run custom jobs. If not specified the project used to run the pipeline will be used. + location: Location used to run custom jobs. If not specified the location used to run the pipeline will be used. + tensorboard_resource_id: Optional tensorboard resource id in format `projects/{project_number}/locations/{location}/tensorboards/{tensorboard_id}`. If provided, tensorboard metrics will be uploaded to this location. + + Returns: + model_resource_name: Path to the model uploaded to the Model Registry. This will be an empty string if the model was not deployed. + endpoint_resource_name: Path the Online Prediction Endpoint. This will be an empty string if the model was not deployed. + """ + # fmt: on + policy_model_lora_dim = 1 + reward_model_lora_dim = 0 + batch_size = 64 + prompt_column = 'input_text' + candidate_columns = ['candidate_0', 'candidate_1'] + choice_column = 'choice' + upload_location = 'us-central1' + machine_spec = function_based.resolve_machine_spec( + location=location, use_test_spec=env.get_use_test_machine_spec() + ).set_display_name('Resolve Machine Spec') + + reference_model_metadata = function_based.resolve_reference_model_metadata( + large_model_reference=large_model_reference, + ).set_display_name('Resolve Model Metadata') + + prompt_dataset_image_uri = function_based.resolve_private_image_uri( + image_name='text_importer' + ).set_display_name('Resolve Prompt Dataset Image URI') + prompt_dataset_importer = ( + private_text_importer.PrivateTextImporter( + project=project, + location=location, + input_text=prompt_dataset, + inputs_field_name=prompt_column, + # Target field name does not matter because this field is not used. + targets_field_name='non_existent_targets_field_name', + output_split_name=env.TRAIN_SPLIT, + large_model_reference=reference_model_metadata.outputs[ + 'large_model_reference' + ], + image_uri=prompt_dataset_image_uri.output, + instruction=instruction, + ) + .set_display_name('Import Prompt Dataset') + .set_caching_options(False) + ) + + preference_dataset_image_uri = function_based.resolve_private_image_uri( + image_name='text_comparison_importer' + ).set_display_name('Resolve Preference Dataset Image URI') + comma_separated_candidates_field_names = ( + function_based.convert_to_delimited_string(items=candidate_columns) + ) + preference_dataset_importer = ( + private_text_comparison_importer.PrivateTextComparisonImporter( + project=project, + location=location, + input_text=preference_dataset, + inputs_field_name=prompt_column, + comma_separated_candidates_field_names=comma_separated_candidates_field_names.output, + choice_field_name=choice_column, + split=env.TRAIN_SPLIT, + large_model_reference=reference_model_metadata.outputs[ + 'reward_model_reference' + ], + image_uri=preference_dataset_image_uri.output, + instruction=instruction, + ) + .set_display_name('Import Preference Dataset') + .set_caching_options(False) + ) + + reward_model_image_uri = function_based.resolve_private_image_uri( + image_name='reward_model', + accelerator_type=machine_spec.outputs['accelerator_type'], + accelerator_count=machine_spec.outputs['accelerator_count'], + ).set_display_name('Resolve Reward Model Image URI') + reward_model = ( + reward_model_trainer.RewardModelTrainer( + project=project, + location=location, + input_model_path=reference_model_metadata.outputs[ + 'reward_model_path' + ], + input_dataset_path=preference_dataset_importer.outputs[ + 'output_dataset_path' + ], + train_steps=reward_model_train_steps, + accelerator_type=machine_spec.outputs['accelerator_type'], + accelerator_count=machine_spec.outputs['accelerator_count'], + large_model_reference=reference_model_metadata.outputs[ + 'reward_model_reference' + ], + machine_type=machine_spec.outputs['machine_type'], + image_uri=reward_model_image_uri.output, + inputs_sequence_length=prompt_sequence_length, + targets_sequence_length=target_sequence_length, + batch_size=batch_size, + learning_rate_multiplier=reward_model_learning_rate_multiplier, + lora_dim=reward_model_lora_dim, + ) + .set_display_name('Reward Model Trainer') + .set_caching_options(False) + ) + + has_tensorboard_id = function_based.value_exists( + value=tensorboard_resource_id + ).set_display_name('Resolve Tensorboard Resource ID') + with kfp.dsl.Condition( # pytype: disable=wrong-arg-types + has_tensorboard_id.output == True, # pylint: disable=singleton-comparison, g-explicit-bool-comparison + name='Upload Reward Model Tensorboard Metrics', + ): + _ = upload_tensorboard_metrics.upload_tensorboard_metrics( + tensorboard_resource_id=tensorboard_resource_id, + metrics_directory=reward_model.outputs['tensorboard_metrics'], + experiment_name=( + 'reward-model-tuner-' + f'{kfp.dsl.PIPELINE_JOB_ID_PLACEHOLDER}-' + f'{kfp.dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + ).set_display_name('Reward Model Tensorboard Metrics Uploader') + + rl_image_uri = function_based.resolve_private_image_uri( + image_name='reinforcer', + accelerator_type=machine_spec.outputs['accelerator_type'], + accelerator_count=machine_spec.outputs['accelerator_count'], + ).set_display_name('Resolve Reinforcer Image URI') + rl_model = ( + reinforcer.Reinforcer( + project=project, + location=location, + input_reference_model_path=reference_model_metadata.outputs[ + 'reference_model_path' + ], + input_reward_model_path=reward_model.outputs['output_model_path'], + input_dataset_path=prompt_dataset_importer.outputs[ + 'imported_data_path' + ], + train_steps=reinforcement_learning_train_steps, + accelerator_type=machine_spec.outputs['accelerator_type'], + accelerator_count=machine_spec.outputs['accelerator_count'], + large_model_reference=reference_model_metadata.outputs[ + 'large_model_reference' + ], + reward_model_reference=reference_model_metadata.outputs[ + 'reward_model_reference' + ], + machine_type=machine_spec.outputs['machine_type'], + image_uri=rl_image_uri.output, + inputs_sequence_length=prompt_sequence_length, + targets_sequence_length=target_sequence_length, + batch_size=batch_size, + learning_rate_multiplier=reinforcement_learning_rate_multiplier, + kl_coeff=kl_coeff, + lora_dim=policy_model_lora_dim, + ) + .set_display_name('Reinforcer') + .set_caching_options(False) + ) + + with kfp.dsl.Condition( # pytype: disable=wrong-arg-types + has_tensorboard_id.output == True, # pylint: disable=singleton-comparison, g-explicit-bool-comparison + name='Upload Reinforcement Learning Tensorboard Metrics', + ): + _ = upload_tensorboard_metrics.upload_tensorboard_metrics( + tensorboard_resource_id=tensorboard_resource_id, + metrics_directory=rl_model.outputs['tensorboard_metrics'], + experiment_name=( + 'rl-model-tuner-' + f'{kfp.dsl.PIPELINE_JOB_ID_PLACEHOLDER}-' + f'{kfp.dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + ).set_display_name('Reinforcement Learning Tensorboard Metrics Uploader') + + should_perform_inference = function_based.value_exists( + value=eval_dataset + ).set_display_name('Resolve Inference Dataset') + with kfp.dsl.Condition( + should_perform_inference.output == True, name='Perform Inference' # pylint: disable=singleton-comparison + ): + component.infer_pipeline( + project=project, + location=location, + large_model_reference=large_model_reference, + model_checkpoint=rl_model.outputs['output_model_path'], + prompt_dataset=eval_dataset, + prompt_sequence_length=prompt_sequence_length, + target_sequence_length=target_sequence_length, + instruction=instruction, + ) + + adapter_artifact = kfp.dsl.importer( + artifact_uri=rl_model.outputs['output_adapter_path'], + artifact_class=kfp.dsl.Artifact, + ).set_display_name('Import Tuned Adapter') + regional_endpoint = function_based.resolve_regional_endpoint( + upload_location=upload_location + ).set_display_name('Resolve Regional Endpoint') + display_name = function_based.resolve_model_display_name( + large_model_reference=reference_model_metadata.outputs[ + 'large_model_reference' + ], + model_display_name=model_display_name, + ).set_display_name('Resolve Model Display Name') + upload_model = function_based.resolve_upload_model( + large_model_reference=reference_model_metadata.outputs[ + 'large_model_reference' + ] + ).set_display_name('Resolve Upload Model') + upload_task = ( + upload_llm_model.upload_llm_model( + project=_placeholders.PROJECT_ID_PLACEHOLDER, + location=upload_location, + regional_endpoint=regional_endpoint.output, + artifact_uri=adapter_artifact.output, + model_display_name=display_name.output, + model_reference_name='text-bison@001', + upload_model=upload_model.output, + ) + .set_env_variable( + name='VERTEX_AI_PIPELINES_RUN_LABELS', + value=json.dumps({'tune-type': 'rlhf'}), + ) + .set_display_name('Upload Model') + ) + deploy_model = function_based.resolve_deploy_model( + deploy_model=deploy_model, + large_model_reference=reference_model_metadata.outputs[ + 'large_model_reference' + ], + ).set_display_name('Resolve Deploy Model') + deploy_task = deploy_llm_model.create_endpoint_and_deploy_model( + project=_placeholders.PROJECT_ID_PLACEHOLDER, + location=upload_location, + model_resource_name=upload_task.outputs['model_resource_name'], + display_name=display_name.output, + regional_endpoint=regional_endpoint.output, + deploy_model=deploy_model.output, + ).set_display_name('Deploy Model') + + return PipelineOutput( + model_resource_name=upload_task.outputs['model_resource_name'], + endpoint_resource_name=deploy_task.outputs['endpoint_resource_name'], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/__init__.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/__init__.py index 366396e11f..6cf10f1d07 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/__init__.py @@ -14,11 +14,17 @@ """Model evaluation preview components.""" from google_cloud_pipeline_components.preview.model_evaluation.data_bias_component import detect_data_bias as DetectDataBiasOp +from google_cloud_pipeline_components.preview.model_evaluation.evaluation_llm_classification_pipeline import evaluation_llm_classification_pipeline +from google_cloud_pipeline_components.preview.model_evaluation.evaluation_llm_text_generation_pipeline import evaluation_llm_text_generation_pipeline from google_cloud_pipeline_components.preview.model_evaluation.feature_attribution_component import feature_attribution as ModelEvaluationFeatureAttributionOp +from google_cloud_pipeline_components.preview.model_evaluation.feature_attribution_graph_component import feature_attribution_graph_component as FeatureAttributionGraphComponentOp from google_cloud_pipeline_components.preview.model_evaluation.model_bias_component import detect_model_bias as DetectModelBiasOp __all__ = [ + 'evaluation_llm_classification_pipeline', + 'evaluation_llm_text_generation_pipeline', 'ModelEvaluationFeatureAttributionOp', + 'FeatureAttributionGraphComponentOp', 'DetectModelBiasOp', 'DetectDataBiasOp', ] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/data_bias_component.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/data_bias_component.py index b2bcf8991a..80f813400a 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/data_bias_component.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/data_bias_component.py @@ -11,8 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from typing import Any, List + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import version from google_cloud_pipeline_components.types.artifact_types import VertexDataset -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG from kfp.dsl import Artifact from kfp.dsl import container_component from kfp.dsl import ContainerSpec @@ -28,15 +32,15 @@ def detect_data_bias( gcp_resources: OutputPath(str), data_bias_metrics: Output[Artifact], - project: str, target_field_name: str, - bias_configs: list, + bias_configs: List[Any], location: str = 'us-central1', dataset_format: str = 'jsonl', - dataset_storage_source_uris: list = [], + dataset_storage_source_uris: List[str] = [], dataset: Input[VertexDataset] = None, - columns: list = [], + columns: List[str] = [], encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Detects data bias metrics in a dataset. @@ -46,61 +50,29 @@ def detect_data_bias( bias metrics for the dataset. Args: - project: Project to run data bias detection. location: Location for running data bias detection. - target_field_name: The full name path of the features target field - in the predictions file. Formatted to be able to find nested columns, - delimited by ``.``. Alternatively referred to as the ground truth (or - ground_truth_column) field. - bias_configs: A list of - ``google.cloud.aiplatform_v1beta1.types.ModelEvaluation.BiasConfig``. - When provided, compute data bias metrics for each defined slice. Below - is an example of how to format this input. - 1: First, create a BiasConfig. - ``from google.cloud.aiplatform_v1beta1.types.ModelEvaluation import BiasConfig`` - - ``from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice import SliceSpec`` + target_field_name: The full name path of the features target field in the predictions file. Formatted to be able to find nested columns, delimited by `.`. Alternatively referred to as the ground truth (or ground_truth_column) field. + bias_configs: A list of `google.cloud.aiplatform_v1beta1.types.ModelEvaluation.BiasConfig`. When provided, compute data bias metrics for each defined slice. Below is an example of how to format this input. - ``from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice.SliceSpec import SliceConfig`` + 1: First, create a BiasConfig. `from google.cloud.aiplatform_v1beta1.types.ModelEvaluation import BiasConfig` `from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice import SliceSpec` `from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice.SliceSpec import SliceConfig` `bias_config = BiasConfig(bias_slices=SliceSpec(configs={ 'feature_a': SliceConfig(SliceSpec.Value(string_value= 'label_a') ) }))` + 2: Create a list to store the bias configs into. `bias_configs = []` + 3: Format each BiasConfig into a JSON or Dict. `bias_config_json = json_format.MessageToJson(bias_config` or `bias_config_dict = json_format.MessageToDict(bias_config).` + 4: Combine each bias_config JSON into a list. `bias_configs.append(bias_config_json)` + 5: Finally, pass bias_configs as an parameter for this component. `DetectDataBiasOp(bias_configs=bias_configs)` - ``bias_config = BiasConfig(bias_slices=SliceSpec(configs={ - 'feature_a': SliceConfig(SliceSpec.Value(string_value= 'label_a') ) }))`` - 2: Create a list to store the bias configs into. - ``bias_configs = []`` - 3: Format each BiasConfig into a JSON or Dict. - ``bias_config_json = json_format.MessageToJson(bias_config`` or - ``bias_config_dict = json_format.MessageToDict(bias_config).`` - 4: Combine each bias_config JSON into a list. - ``bias_configs.append(bias_config_json)`` - 5: Finally, pass bias_configs as an parameter for this component. - ``DetectDataBiasOp(bias_configs=bias_configs)`` - dataset_format: The file format for the dataset. ``jsonl`` and ``csv`` are the - currently allowed formats. - dataset_storage_source_uris: Google Cloud - Storage URI(-s) to unmanaged test datasets.``jsonl`` and ``csv`` is currently - allowed format. If ``dataset`` is also provided, this field will be - overriden by the provided Vertex Dataset. - dataset: A ``google.VertexDataset`` - artifact of the dataset. If ``dataset_gcs_source`` is also provided, this - Vertex Dataset argument will override the GCS source. - encryption_spec_key_name: Customer-managed encryption key options for the - Dataflow. If this is set, then all resources created by the Dataflow - will be encrypted with the provided encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. + dataset_format: The file format for the dataset. `jsonl` and `csv` are the currently allowed formats. + dataset_storage_source_uris: Google Cloud Storage URI(-s) to unmanaged test datasets.`jsonl` and `csv` is currently allowed format. If `dataset` is also provided, this field will be overriden by the provided Vertex Dataset. + dataset: A `google.VertexDataset` artifact of the dataset. If `dataset_gcs_source` is also provided, this Vertex Dataset argument will override the GCS source. + encryption_spec_key_name: Customer-managed encryption key options for the Dataflow. If this is set, then all resources created by the Dataflow will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + project: Project to run data bias detection. Defaults to the project in which the PipelineJob is run. Returns: - data_bias_metrics: - Artifact tracking the data bias detection output. - gcp_resources: - Serialized gcp_resources proto tracking the Dataflow job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + data_bias_metrics: Artifact tracking the data bias detection output. + gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_classification_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_classification_pipeline.py new file mode 100644 index 0000000000..e013db0f1e --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_classification_pipeline.py @@ -0,0 +1,164 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Vertex Gen AI Evaluation for text classification task.""" + +from typing import Dict, List, NamedTuple + +from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationClassificationPredictionsPostprocessorOp +from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp +from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics +from google_cloud_pipeline_components.types.artifact_types import VertexModel +from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp +from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp +from kfp import dsl + +_PIPELINE_NAME = 'evaluation-llm-classification-pipeline' + + +@dsl.pipeline(name=_PIPELINE_NAME) +def evaluation_llm_classification_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + target_field_name: str, + batch_predict_gcs_source_uris: List[str], + batch_predict_gcs_destination_output_uri: str, + model_name: str = 'publishers/google/models/text-bison@001', + evaluation_task: str = 'text-classification', + evaluation_class_labels: List[str] = [], + batch_predict_instances_format: str = 'jsonl', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_model_parameters: Dict[str, str] = {}, + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + dataflow_machine_type: str = 'n1-standard-4', + dataflow_disk_size_gb: int = 50, + dataflow_max_num_workers: int = 5, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-llm-classification-pipeline-{{$.pipeline_job_uuid}}', +) -> NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, +): + # fmt: off + """The LLM Text Classification Evaluation pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. + model_name: The Model name used to run evaluation. Must be a publisher Model or a managed Model sharing the same ancestor location. Starting this job has no impact on any existing deployments of the Model and their resources. + evaluation_task: The task that the large language model will be evaluated on. The evaluation component computes a set of metrics relevant to that specific task. Currently supported Classification tasks is: `text-classification`. + evaluation_class_labels: The JSON array of class names for the target_field, in the same order they appear in the batch predictions input file. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_model_parameters: A map of parameters that govern the predictions. Some acceptable parameters include: maxOutputTokens, topK, topP, and temperature. + machine_type: The machine type of the custom jobs in this pipeline. If not set, defaulted to `e2-highmem-16`. More details: https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as account. The service account running the pipeline (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code Service Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name, as in `myVPC`. To specify this field, you must have already configured VPC Network Peering for Vertex AI (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left unspecified, the job is not peered with any network. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. If not set, defaulted to `50`. + dataflow_max_num_workers: The max number of workers executing the evaluation run. If not set, defaulted to `5`. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + + Returns: + evaluation_metrics: ClassificationMetrics Artifact for LLM Text Classification. + evaluation_resource_name: If run on an user's managed VertexModel, the imported evaluation resource name. Empty if run on a publisher model. + """ + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, + ) + + get_vertex_model_task = dsl.importer( + artifact_uri=( + f'https://{location}-aiplatform.googleapis.com/v1/{model_name}' + ), + artifact_class=VertexModel, + metadata={'resourceName': model_name}, + ) + get_vertex_model_task.set_display_name('get-vertex-model') + + batch_predict_task = ModelBatchPredictOp( + project=project, + location=location, + model=get_vertex_model_task.outputs['artifact'], + job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=batch_predict_gcs_source_uris, + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, + model_parameters=batch_predict_model_parameters, + encryption_spec_key_name=encryption_spec_key_name, + ) + + postprocessor_task = LLMEvaluationClassificationPredictionsPostprocessorOp( + project=project, + batch_prediction_results=batch_predict_task.outputs[ + 'gcs_output_directory' + ], + class_labels=evaluation_class_labels, + location=location, + machine_type=machine_type, + network=network, + service_account=service_account, + encryption_spec_key_name=encryption_spec_key_name, + ) + + eval_task = ModelEvaluationClassificationOp( + project=project, + location=location, + class_labels=postprocessor_task.outputs['postprocessed_class_labels'], + target_field_name=target_field_name, + predictions_gcs_source=postprocessor_task.outputs[ + 'postprocessed_predictions_gcs_source' + ], + prediction_label_column='prediction.classes', + prediction_score_column='prediction.scores', + predictions_format=batch_predict_predictions_format, + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + ) + + import_evaluation_task = ModelImportEvaluationOp( + classification_metrics=eval_task.outputs['evaluation_metrics'], + model=get_vertex_model_task.outputs['artifact'], + dataset_type=batch_predict_instances_format, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_text_generation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_text_generation_pipeline.py new file mode 100644 index 0000000000..cae0f06a13 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/evaluation_llm_text_generation_pipeline.py @@ -0,0 +1,131 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Vertex Gen AI Evaluation for Text Generation/QA/Summarization tasks.""" + +from typing import Dict, List, NamedTuple + +from google_cloud_pipeline_components._implementation.model_evaluation import LLMEvaluationTextGenerationOp +from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp +from google_cloud_pipeline_components.types.artifact_types import VertexModel +from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp +from kfp import dsl + +_PIPELINE_NAME = 'evaluation-llm-text-generation-pipeline' + + +@dsl.pipeline(name=_PIPELINE_NAME) +def evaluation_llm_text_generation_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + batch_predict_gcs_source_uris: List[str], + batch_predict_gcs_destination_output_uri: str, + batch_predict_model_parameters: Dict[str, str] = {}, + model_name: str = 'publishers/google/models/text-bison@001', + evaluation_task: str = 'text-generation', + batch_predict_instances_format: str = 'jsonl', + batch_predict_predictions_format: str = 'jsonl', + machine_type: str = 'e2-highmem-16', + service_account: str = '', + network: str = '', + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-llm-text-generation-pipeline-{{$.pipeline_job_uuid}}', +) -> NamedTuple( + 'outputs', evaluation_metrics=dsl.Metrics, evaluation_resource_name=str +): + # fmt: off + """LLM Text Generation Evaluation pipeline. + + This pipeline supports evaluating large language models, publisher or managed + models, performing the following generative tasks: `summarization`, `question-answering`, and `text-generation`. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. + model_name: The Model name used to run evaluation. Must be a publisher Model or a managed Model sharing the same ancestor location. Starting this job has no impact on any existing deployments of the Model and their resources. + evaluation_task: The task that the large language model will be evaluated on. The evaluation component computes a set of metrics relevant to that specific task. Currently supported tasks are: `summarization`, `question-answering`, `text-generation`. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. Only "jsonl" is currently supported. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. Only "jsonl" is currently supported. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_model_parameters: A map of parameters that govern the predictions. Some acceptable parameters include: maxOutputTokens, topK, topP, and temperature. + machine_type: The machine type of this custom job. If not set, defaulted to `e2-highmem-16`. More details: https://cloud.google.com/compute/docs/machine-resource + service_account: Sets the default service account for workload run-as account. The service account running the pipeline (https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code Service Agent(https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name, as in `myVPC`. To specify this field, you must have already configured VPC Network Peering for Vertex AI (https://cloud.google.com/vertex-ai/docs/general/vpc-peering). If left unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + + Returns: + evaluation_metrics: Metrics Artifact for LLM Text Generation. + evaluation_resource_name: If run on an user's managed VertexModel, the imported evaluation resource name. Empty if run on a publisher model. + """ + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=dsl.Metrics, + evaluation_resource_name=str, + ) + + get_vertex_model_task = dsl.importer( + artifact_uri=( + f'https://{location}-aiplatform.googleapis.com/v1/{model_name}' + ), + artifact_class=VertexModel, + metadata={'resourceName': model_name}, + ) + get_vertex_model_task.set_display_name('get-vertex-model') + + batch_predict_task = ModelBatchPredictOp( + project=project, + location=location, + model=get_vertex_model_task.outputs['artifact'], + job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=batch_predict_gcs_source_uris, + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, + model_parameters=batch_predict_model_parameters, + encryption_spec_key_name=encryption_spec_key_name, + ) + + eval_task = LLMEvaluationTextGenerationOp( + project=project, + location=location, + evaluation_task=evaluation_task, + target_field_name='instance.ground_truth', + prediction_field_name='predictions.content', + predictions_format=batch_predict_predictions_format, + joined_predictions_gcs_source=batch_predict_task.outputs[ + 'gcs_output_directory' + ], + machine_type=machine_type, + service_account=service_account, + network=network, + encryption_spec_key_name=encryption_spec_key_name, + ) + + import_evaluation_task = ModelImportEvaluationOp( + metrics=eval_task.outputs['evaluation_metrics'], + model=get_vertex_model_task.outputs['artifact'], + problem_type=evaluation_task, + dataset_type=batch_predict_predictions_format, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_component.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_component.py index d9e72f7fd3..2081b7a4dc 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_component.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_component.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import version from google_cloud_pipeline_components.types.artifact_types import BQTable -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component @@ -32,7 +33,6 @@ def feature_attribution( gcp_resources: OutputPath(str), feature_attributions: Output[Metrics], - project: str, problem_type: str, location: str = 'us-central1', predictions_format: str = 'jsonl', @@ -47,6 +47,7 @@ def feature_attribution( dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Compute feature attribution on a trained model's batch explanation @@ -57,57 +58,30 @@ def feature_attribution( possible, typically possible for AutoML Classification models. Args: - project: Project to run feature attribution container. - location: Location running feature attribution. If not - set, defaulted to `us-central1`. - problem_type: Problem type of the pipeline: one of `classification`, - `regression` and `forecasting`. - predictions_format: The file format for the batch - prediction results. `jsonl`, `csv`, and `bigquery` are the allowed - formats, from Vertex Batch Prediction. If not set, defaulted to `jsonl`. - predictions_gcs_source: An artifact with its - URI pointing toward a GCS directory with prediction or explanation files - to be used for this evaluation. For prediction results, the files should - be named "prediction.results-*" or "predictions_". For explanation - results, the files should be named "explanation.results-*". - predictions_bigquery_source: BigQuery table - with prediction or explanation data to be used for this evaluation. For - prediction results, the table column should be named "predicted_*". - dataflow_service_account: Service account to run the - dataflow job. If not set, dataflow will use the default worker service - account. For more details, see - https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account - dataflow_disk_size_gb: The disk size (in GB) of the machine - executing the evaluation run. If not set, defaulted to `50`. - dataflow_machine_type: The machine type executing the - evaluation run. If not set, defaulted to `n1-standard-4`. - dataflow_workers_num: The number of workers executing the - evaluation run. If not set, defaulted to `10`. - dataflow_max_workers_num: The max number of workers - executing the evaluation run. If not set, defaulted to `25`. - dataflow_subnetwork: Dataflow's fully qualified subnetwork - name, when empty the default subnetwork will be used. More details: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow - workers use public IP addresses. - encryption_spec_key_name: Customer-managed encryption key - for the Dataflow job. If this is set, then all resources created by the - Dataflow job will be encrypted with the provided encryption key. - force_runner_mode: Flag to choose Beam runner. Valid options are `DirectRunner` - and `Dataflow`. + location: Location running feature attribution. If not set, defaulted to `us-central1`. + problem_type: Problem type of the pipeline: one of `classification`, `regression` and `forecasting`. + predictions_format: The file format for the batch prediction results. `jsonl`, `csv`, and `bigquery` are the allowed formats, from Vertex Batch Prediction. If not set, defaulted to `jsonl`. + predictions_gcs_source: An artifact with its URI pointing toward a GCS directory with prediction or explanation files to be used for this evaluation. For prediction results, the files should be named "prediction.results-*" or "predictions_". For explanation results, the files should be named "explanation.results-*". + predictions_bigquery_source: BigQuery table with prediction or explanation data to be used for this evaluation. For prediction results, the table column should be named "predicted_*". + dataflow_service_account: Service account to run the dataflow job. If not set, dataflow will use the default worker service account. For more details, see https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. If not set, defaulted to `50`. + dataflow_machine_type: The machine type executing the evaluation run. If not set, defaulted to `n1-standard-4`. + dataflow_workers_num: The number of workers executing the evaluation run. If not set, defaulted to `10`. + dataflow_max_workers_num: The max number of workers executing the evaluation run. If not set, defaulted to `25`. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key for the Dataflow job. If this is set, then all resources created by the Dataflow job will be encrypted with the provided encryption key. + force_runner_mode: Flag to choose Beam runner. Valid options are `DirectRunner` and `Dataflow`. + project: Project to run feature attribution container. Defaults to the project in which the PipelineJob is run. Returns: - gcs_output_directory: JsonArray of the downsampled dataset GCS - output. - bigquery_output_table: String of the downsampled dataset BigQuery - output. - gcp_resources: Serialized gcp_resources proto tracking the dataflow - job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcs_output_directory: JsonArray of the downsampled dataset GCS output. + bigquery_output_table: String of the downsampled dataset BigQuery output. + gcp_resources: Serialized gcp_resources proto tracking the dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_graph_component.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_graph_component.py new file mode 100644 index 0000000000..5e587d9216 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/feature_attribution_graph_component.py @@ -0,0 +1,154 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, NamedTuple + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import EvaluationDataSamplerOp +from google_cloud_pipeline_components.preview.model_evaluation.feature_attribution_component import feature_attribution as ModelEvaluationFeatureAttributionOp +from google_cloud_pipeline_components.types.artifact_types import VertexModel +from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp +import kfp + + +@kfp.dsl.pipeline(name='feature-attribution-graph-component') +def feature_attribution_graph_component( # pylint: disable=dangerous-default-value + location: str, + prediction_type: str, + vertex_model: VertexModel, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_explanation_metadata: dict = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: dict = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_data_sample_size: int = 10000, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +) -> NamedTuple('outputs', feature_attributions=kfp.dsl.Metrics): + # fmt: off + """A pipeline to compute feature attributions by sampling data for batch explanations. + + This pipeline guarantees support for AutoML Tabular models that contain a + valid explanation_spec. + + Args: + location: The GCP region that runs the pipeline components. + prediction_type: The type of prediction the model is to produce. "classification", "regression", or "forecasting". + vertex_model: The Vertex model artifact used for batch explanation. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the project in which the PipelineJob is run. + + Returns: + A system.Metrics artifact with feature attributions. + """ + # fmt: øn + outputs = NamedTuple('outputs', feature_attributions=kfp.dsl.Metrics) + + # Sample the input dataset for a quicker batch explanation. + data_sampler_task = EvaluationDataSamplerOp( + project=project, + location=location, + gcs_source_uris=batch_predict_gcs_source_uris, + bigquery_source_uri=batch_predict_bigquery_source_uri, + instances_format=batch_predict_instances_format, + sample_size=batch_predict_explanation_data_sample_size, + force_runner_mode=force_runner_mode, + ) + + # Run batch explain. + batch_explain_task = ModelBatchPredictOp( + project=project, + location=location, + model=vertex_model, + job_display_name='model-registry-batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=data_sampler_task.outputs['gcs_output_directory'], + bigquery_source_input_uri=data_sampler_task.outputs[ + 'bigquery_output_table' + ], + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, + bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + generate_explanation=True, + explanation_parameters=batch_predict_explanation_parameters, + explanation_metadata=batch_predict_explanation_metadata, + machine_type=batch_predict_machine_type, + starting_replica_count=batch_predict_starting_replica_count, + max_replica_count=batch_predict_max_replica_count, + encryption_spec_key_name=encryption_spec_key_name, + accelerator_type=batch_predict_accelerator_type, + accelerator_count=batch_predict_accelerator_count, + ) + + # Generate feature attributions from explanations. + feature_attribution_task = ModelEvaluationFeatureAttributionOp( + project=project, + location=location, + problem_type=prediction_type, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_explain_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_explain_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + ) + + return outputs( + feature_attributions=feature_attribution_task.outputs[ + 'feature_attributions' + ] + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_bias_component.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_bias_component.py index 2295ee1480..5489e91cc1 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_bias_component.py +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/model_bias_component.py @@ -11,8 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import version from google_cloud_pipeline_components.types.artifact_types import BQTable -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component @@ -30,7 +31,6 @@ def detect_model_bias( gcp_resources: OutputPath(str), bias_model_metrics: Output[Artifact], - project: str, target_field_name: str, bias_configs: list, location: str = 'us-central1', @@ -39,6 +39,7 @@ def detect_model_bias( predictions_bigquery_source: Input[BQTable] = None, thresholds: list = [0.5], encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Detects bias metrics from a model's predictions. @@ -48,64 +49,30 @@ def detect_model_bias( bias metrics for classification problems. Args: - project: Project to run data bias detection. location: Location for running data bias detection. - target_field_name: The full name path of the features target field in the - predictions file. Formatted to be able to find nested columns, delimited - by ``.``. Alternatively referred to as the ground truth (or - ground_truth_column) field. - predictions_format: The file format for the batch prediction results. - ``jsonl`` is the only currently allow format. - predictions_gcs_source: An artifact with its URI pointing toward a GCS - directory with prediction or explanation files to be used for this - evaluation. For prediction results, the files should be named - "prediction.results-*". For explanation results, the files should be - named "explanation.results-*". - predictions_bigquery_source: BigQuery table with prediction or explanation - data to be used for this evaluation. For prediction results, the table - column should be named "predicted_*". - bias_configs: A list of - ``google.cloud.aiplatform_v1beta1.types.ModelEvaluation.BiasConfig``. - When provided, compute model bias metrics for each defined slice. Below - is an example of how to format this input. - 1: First, create a BiasConfig. - ``from google.cloud.aiplatform_v1beta1.types.ModelEvaluation import BiasConfig`` + target_field_name: The full name path of the features target field in the predictions file. Formatted to be able to find nested columns, delimited by `.`. Alternatively referred to as the ground truth (or ground_truth_column) field. + predictions_format: The file format for the batch prediction results. `jsonl` is the only currently allow format. + predictions_gcs_source: An artifact with its URI pointing toward a GCS directory with prediction or explanation files to be used for this evaluation. For prediction results, the files should be named "prediction.results-*". For explanation results, the files should be named "explanation.results-*". + predictions_bigquery_source: BigQuery table with prediction or explanation data to be used for this evaluation. For prediction results, the table column should be named "predicted_*". + bias_configs: A list of `google.cloud.aiplatform_v1beta1.types.ModelEvaluation.BiasConfig`. When provided, compute model bias metrics for each defined slice. Below is an example of how to format this input. - ``from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice import SliceSpec`` + 1: First, create a BiasConfig. `from google.cloud.aiplatform_v1beta1.types.ModelEvaluation import BiasConfig` `from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice import SliceSpec` `from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice.SliceSpec import SliceConfig` `bias_config = BiasConfig(bias_slices=SliceSpec(configs={ 'feature_a': SliceConfig(SliceSpec.Value(string_value= 'label_a') ) }))` + 2: Create a list to store the bias configs into. `bias_configs = []` + 3: Format each BiasConfig into a JSON or Dict. `bias_config_json = json_format.MessageToJson(bias_config` or `bias_config_dict = json_format.MessageToDict(bias_config)` + 4: Combine each bias_config JSON into a list. `bias_configs.append(bias_config_json)` + 5: Finally, pass bias_configs as an parameter for this component. `DetectModelBiasOp(bias_configs=bias_configs)` - ``from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice.SliceSpec import SliceConfig`` - - ``bias_config = BiasConfig(bias_slices=SliceSpec(configs={ - 'feature_a': SliceConfig(SliceSpec.Value(string_value= 'label_a') ) }))`` - 2: Create a list to store the bias configs into. - ``bias_configs = []`` - 3: Format each BiasConfig into a JSON or Dict. - ``bias_config_json = json_format.MessageToJson(bias_config`` or - ``bias_config_dict = json_format.MessageToDict(bias_config)`` - 4: Combine each bias_config JSON into a list. - ``bias_configs.append(bias_config_json)`` - 5: Finally, pass bias_configs as an parameter for this component. - ``DetectModelBiasOp(bias_configs=bias_configs)`` - thresholds: A list of float values to be used as prediction decision - thresholds. - encryption_spec_key_name: Customer-managed encryption key options for the - Dataflow. If this is set, then all resources created by the Dataflow - will be encrypted with the provided encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. + thresholds: A list of float values to be used as prediction decision thresholds. + encryption_spec_key_name: Customer-managed encryption key options for the Dataflow. If this is set, then all resources created by the Dataflow will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + project: Project to run data bias detection. Defaults to the project in which the PipelineJob is run. Returns: - bias_model_metrics: - Artifact tracking the model bias detection output. - gcp_resources: - Serialized gcp_resources proto tracking the Dataflow job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + bias_model_metrics: Artifact tracking the model bias detection output. + gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/utils.py b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/utils.py new file mode 100644 index 0000000000..d235543250 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/preview/model_evaluation/utils.py @@ -0,0 +1,183 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Union + +from google.cloud.aiplatform_v1.types.model_evaluation_slice import ModelEvaluationSlice + +from google.protobuf.wrappers_pb2 import BoolValue +from google.protobuf import json_format + + +def create_slice_specs_list( + list_of_feature_and_value: List[ + Dict[str, Union[float, int, str, List[float], bool]] + ] +) -> List[ModelEvaluationSlice.Slice.SliceSpec]: + """Creates a list of ModelEvaluationSlice.Slice.SliceSpec from a list of dictionary inputs. + + Args: + list_of_feature_and_value: A list of feature_and_value. Each + feature_and_value is a dictionary of feature names to values. The feature + value can be a float, int, or str for + ModelEvaluationSlice.Slice.SliceSpec.Value; a bool for `all_values` or a + list for ModelEvaluationSlice.Slice.SliceSpec.Range. + + Returns: A list of ModelEvaluationSlice.Slice.SliceSpec proto. + + Raises: + ValueError: if the format of a feature's value is invalid. + """ + slice_specs_list = [] + for feature_and_value in list_of_feature_and_value: + configs = {} + for feature, value in feature_and_value.items(): + if isinstance(value, bool): + # Bool must be checked first, bool is a child of int in Python. + configs[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + all_values=BoolValue(value=value) + ) + elif isinstance(value, int) or isinstance(value, float): + configs[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + value=ModelEvaluationSlice.Slice.SliceSpec.Value( + float_value=float(value) + ) + ) + elif isinstance(value, str): + configs[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + value=ModelEvaluationSlice.Slice.SliceSpec.Value(string_value=value) + ) + elif isinstance(value, list): + configs[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + range_=ModelEvaluationSlice.Slice.SliceSpec.Range( + low=value[0], high=value[1] + ) + ) + else: + raise ValueError( + 'Please provide a valid format of value for feature: {}. The' + ' accepted formats are: bool, float, int, str and list.'.format( + feature + ) + ) + slice_spec = ModelEvaluationSlice.Slice.SliceSpec(configs=configs) + slice_specs_list.append(json_format.MessageToDict(slice_spec._pb)) + return slice_specs_list + + +def create_bias_configs_list( + list_of_slice_a_and_slice_b: List[ + List[Dict[str, Union[float, int, str, List[float]]]] + ], +) -> List[Any]: + """Creates a list of BiasConfig from a list of tuple inputs. + + Args: + list_of_slice_a_and_slice_b: A list of slice_a_and_slice_b. Each + slice_a_and_slice_b is a list which contains 1 or two elelments. Each + element in the list is a dictionary of feature names to values that + represents the slice config for 'slice_a' or 'slice_b'. 'slice_b' is + optional. The feature value can be a float, int, or str for + ModelEvaluationSlice.Slice.SliceSpec.Value; a list for + ModelEvaluationSlice.Slice.SliceSpec.Range. Following are example inputs: + Ex 1. Only provide the config of slice_a: `list_of_slice_a_and_slice_b = + [[{'education': 'low'}]]`. Ex 2. Provide both configs of slice_a and + slice_b: `list_of_slice_a_and_slice_b = [[{'education': 'low'}, + {'education': 'high'}]]`. + + Returns: A list of BiasConfig. + + Raises: + ValueError: if a feature's value is `all_values` or the format of the + feature's value is invalid. + """ + bias_configs_list = [] + for slice_a_and_slice_b in list_of_slice_a_and_slice_b: + slice_a = slice_a_and_slice_b[0] + if len(slice_a_and_slice_b) > 1: + slice_b = slice_a_and_slice_b[1] + else: + slice_b = None + bias_config = {} + configs_a = {} + for feature, value in slice_a.items(): + if isinstance(value, bool): + # Bool must be checked first, bool is a child of int in Python. + raise ValueError( + '`all_values` SliceConfig is not allowed for bias detection.' + ) + elif isinstance(value, int) or isinstance(value, float): + configs_a[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + value=ModelEvaluationSlice.Slice.SliceSpec.Value( + float_value=float(value) + ) + ) + elif isinstance(value, str): + configs_a[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + value=ModelEvaluationSlice.Slice.SliceSpec.Value(string_value=value) + ) + elif isinstance(value, list): + configs_a[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + range=ModelEvaluationSlice.Slice.SliceSpec.Range( + low=value[0], high=value[1] + ) + ) + else: + raise ValueError( + 'Please provide a valid format of value for feature: {}. The' + ' accepted formats are: bool, float, int, str and list.'.format( + feature + ) + ) + slice_spec_a = ModelEvaluationSlice.Slice.SliceSpec(configs=configs_a) + slice_spec_a_dict = json_format.MessageToDict(slice_spec_a._pb) + bias_config['slices'] = [slice_spec_a_dict] + if slice_b is not None: + configs_b = {} + for feature, value in slice_b.items(): + if isinstance(value, bool): + # Bool must be checked first, bool is a child of int in Python. + raise ValueError( + '`all_values` SliceConfig is not allowed for bias detection.' + ) + elif isinstance(value, int) or isinstance(value, float): + configs_b[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + value=ModelEvaluationSlice.Slice.SliceSpec.Value( + float_value=float(value) + ) + ) + elif isinstance(value, str): + configs_b[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + value=ModelEvaluationSlice.Slice.SliceSpec.Value( + string_value=value + ) + ) + elif isinstance(value, list): + configs_b[feature] = ModelEvaluationSlice.Slice.SliceSpec.SliceConfig( + range=ModelEvaluationSlice.Slice.SliceSpec.Range( + low=value[0], high=value[1] + ) + ) + else: + raise ValueError( + 'Please provide a valid format of value for feature: {}. The' + ' accepted formats are: bool, float, int, str and list.'.format( + feature + ) + ) + slice_spec_b = ModelEvaluationSlice.Slice.SliceSpec(configs=configs_b) + slice_spec_b_dict = json_format.MessageToDict(slice_spec_b._pb) + bias_config['slices'].append(slice_spec_b_dict) + bias_configs_list.append(bias_config) + return bias_configs_list diff --git a/components/google-cloud/google_cloud_pipeline_components/proto/README.md b/components/google-cloud/google_cloud_pipeline_components/proto/README.md index 6b1dce053a..2162abb80d 100644 --- a/components/google-cloud/google_cloud_pipeline_components/proto/README.md +++ b/components/google-cloud/google_cloud_pipeline_components/proto/README.md @@ -14,7 +14,7 @@ pip install -U google-cloud-pipeline-components To write a resource as an output parameter ``` -from google_cloud_pipeline_components.experimental.proto.gcp_resources_pb2 import GcpResources +from google_cloud_pipeline_components.proto.gcp_resources_pb2 import GcpResources from google.protobuf.json_format import MessageToJson dataflow_resources = GcpResources() diff --git a/components/google-cloud/google_cloud_pipeline_components/types/artifact_types.py b/components/google-cloud/google_cloud_pipeline_components/types/artifact_types.py index c05920043d..9aef1f37d6 100644 --- a/components/google-cloud/google_cloud_pipeline_components/types/artifact_types.py +++ b/components/google-cloud/google_cloud_pipeline_components/types/artifact_types.py @@ -14,13 +14,13 @@ """Artifact types corresponding to Google Cloud Resources produced and consumed by GCPC components. These artifact types can be used in your custom KFP SDK components similarly to -other `KFP SDK artifacts -`_. +other [KFP SDK +artifacts](https://www.kubeflow.org/docs/components/pipelines/v2/data-types/artifacts/). If you wish to produce Google artifacts from your own components, it is -recommended that you use `Containerized Python Components -`_. +recommended that you use [Containerized Python +Components](https://www.kubeflow.org/docs/components/pipelines/v2/components/containerized-python-components/). You should assign metadata to the Google artifacts according to the artifact's -schema (provided by each artifact's ``.schema`` attribute). +schema (provided by each artifact's `.schema` attribute). """ @@ -38,14 +38,15 @@ ] import textwrap -from typing import Dict, Optional +from typing import Any, Dict, Optional + from kfp import dsl _RESOURCE_NAME_KEY = 'resourceName' class VertexModel(dsl.Artifact): - """An artifact representing a Vertex AI `Model resource `_.""" + """An artifact representing a Vertex AI [Model resource](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models).""" schema_title = 'google.VertexModel' schema_version = '0.0.1' schema = textwrap.dedent("""\ @@ -62,19 +63,18 @@ def create( uri: str, model_resource_name: str, ) -> 'VertexModel': + # fmt: off """Create a VertexModel artifact instance. Args: name: The artifact name. - uri: the Vertex Model resource uri, in a form of - https://{service-endpoint}/v1/projects/{project}/locations/{location}/models/{model}, - where {service-endpoint} is one of the supported service endpoints at - https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints - model_resource_name: The name of the Model resource, in a form of - projects/{project}/locations/{location}/models/{model}. For more - details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/get + uri: the Vertex Model resource uri, in a form of https://{service-endpoint}/v1/projects/{project}/locations/{location}/models/{model}, where {service-endpoint} is one of the supported service endpoints at https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints + model_resource_name: The name of the Model resource, in a form of projects/{project}/locations/{location}/models/{model}. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/get + + Returns: + VertexModel instance. """ + # fmt: on return cls( name=name, uri=uri, @@ -83,7 +83,7 @@ def create( class VertexEndpoint(dsl.Artifact): - """An artifact representing a Vertex AI `Endpoint resource `_.""" + """An artifact representing a Vertex AI [Endpoint resource](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints).""" schema_title = 'google.VertexEndpoint' schema_version = '0.0.1' schema = textwrap.dedent("""\ @@ -100,19 +100,18 @@ def create( uri: str, endpoint_resource_name: str, ) -> 'VertexEndpoint': + # fmt: off """Create a VertexEndpoint artifact instance. Args: name: The artifact name. - uri: the Vertex Endpoint resource uri, in a form of - https://{service-endpoint}/v1/projects/{project}/locations/{location}/endpoints/{endpoint}, - where {service-endpoint} is one of the supported service endpoints at - https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints - endpoint_resource_name: The name of the Endpoint resource, in a form of - projects/{project}/locations/{location}/endpoints/{endpoint}. For more - details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/get + uri: the Vertex Endpoint resource uri, in a form of https://{service-endpoint}/v1/projects/{project}/locations/{location}/endpoints/{endpoint}, where {service-endpoint} is one of the supported service endpoints at https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints + endpoint_resource_name: The name of the Endpoint resource, in a form of projects/{project}/locations/{location}/endpoints/{endpoint}. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/get + + Returns: + VertexEndpoint instance. """ + # fmt: on return cls( name=name, uri=uri, @@ -121,7 +120,7 @@ def create( class VertexBatchPredictionJob(dsl.Artifact): - """An artifact representing a Vertex AI `BatchPredictionJob resource `_.""" + """An artifact representing a Vertex AI [BatchPredictionJob resource](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#resource:-batchpredictionjob).""" schema_title = 'google.VertexBatchPredictionJob' schema_version = '0.0.1' schema = textwrap.dedent("""\ @@ -147,32 +146,21 @@ def create( bigquery_output_dataset: Optional[str] = None, gcs_output_directory: Optional[str] = None, ) -> 'VertexBatchPredictionJob': + # fmt: off """Create a VertexBatchPredictionJob artifact instance. Args: name: The artifact name. - uri: the Vertex Batch Prediction resource uri, in a form of - https://{service-endpoint}/v1/projects/{project}/locations/{location}/batchPredictionJobs/{batchPredictionJob}, - where {service-endpoint} is one of the supported service endpoints at - https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints - job_resource_name: The name of the batch prediction job resource, in a - form of - projects/{project}/locations/{location}/batchPredictionJobs/{batchPredictionJob}. - For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs/get - bigquery_output_table: The name of the BigQuery table created, in - predictions_ format, into which the prediction output is - written. For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#outputinfo - bigquery_output_dataset: The path of the BigQuery dataset created, in - bq://projectId.bqDatasetId format, into which the prediction output is - written. For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#outputinfo - gcs_output_directory: The full path of the Cloud Storage directory - created, into which the prediction output is written. For more details, - see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#outputinfo + uri: the Vertex Batch Prediction resource uri, in a form of https://{service-endpoint}/v1/projects/{project}/locations/{location}/batchPredictionJobs/{batchPredictionJob}, where {service-endpoint} is one of the supported service endpoints at https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints + job_resource_name: The name of the batch prediction job resource, in a form of projects/{project}/locations/{location}/batchPredictionJobs/{batchPredictionJob}. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs/get + bigquery_output_table: The name of the BigQuery table created, in predictions_ format, into which the prediction output is written. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#outputinfo + bigquery_output_dataset: The path of the BigQuery dataset created, in bq://projectId.bqDatasetId format, into which the prediction output is written. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#outputinfo + gcs_output_directory: The full path of the Cloud Storage directory created, into which the prediction output is written. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#outputinfo + + Returns: + VertexBatchPredictionJob instance. """ + # fmt: on return cls( name=name, uri=uri, @@ -186,7 +174,7 @@ def create( class VertexDataset(dsl.Artifact): - """An artifact representing a Vertex AI `Dataset resource `_.""" + """An artifact representing a Vertex AI [Dataset resource](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets).""" schema_title = 'google.VertexDataset' schema_version = '0.0.1' schema = textwrap.dedent("""\ @@ -203,18 +191,16 @@ def create( uri: str, dataset_resource_name: str, ) -> 'VertexDataset': + # fmt: off """Create a VertexDataset artifact instance. Args: name: The artifact name. - uri: the Vertex Dataset resource uri, in a form of - https://{service-endpoint}/v1/projects/{project}/locations/{location}/datasets/{datasets_name}, - where {service-endpoint} is one of the supported service endpoints at - https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints - dataset_resource_name: The name of the Dataset resource, in a form of - projects/{project}/locations/{location}/datasets/{datasets_name}. For - more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets/get + uri: the Vertex Dataset resource uri, in a form of https://{service-endpoint}/v1/projects/{project}/locations/{location}/datasets/{datasets_name}, where {service-endpoint} is one of the supported service endpoints at https://cloud.google.com/vertex-ai/docs/reference/rest#rest_endpoints + dataset_resource_name: The name of the Dataset resource, in a form of projects/{project}/locations/{location}/datasets/{datasets_name}. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets/get + + Returns: + VertexDataset instance. """ return cls( uri=uri, @@ -224,7 +210,7 @@ def create( class BQMLModel(dsl.Artifact): - """An artifact representing a Google Cloud `BQML Model resource `_.""" + """An artifact representing a Google Cloud [BQML Model resource](https://cloud.google.com/bigquery/docs/reference/rest/v2/models).""" schema_title = 'google.BQMLModel' schema_version = '0.0.1' schema = textwrap.dedent("""\ @@ -246,15 +232,19 @@ def create( dataset_id: str, model_id: str, ) -> 'BQMLModel': + # fmt: off """Create a BQMLModel artifact instance. Args: name: The artifact name. project_id: The ID of the project containing this model. dataset_id: The ID of the dataset containing this model. - model_id: The ID of the model. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/models#ModelReference + model_id: The ID of the model. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/models#ModelReference + + Returns: + BQMLModel instance. """ + # fmt: on return cls( name=name, uri=f'https://www.googleapis.com/bigquery/v2/projects/{project_id}/datasets/{dataset_id}/models/{model_id}', @@ -267,7 +257,7 @@ def create( class BQTable(dsl.Artifact): - """An artifact representing a Google Cloud `BQ Table resource `_.""" + """An artifact representing a Google Cloud [BQ Table resource](https://cloud.google.com/bigquery/docs/reference/rest/v2/tables).""" schema_title = 'google.BQTable' schema_version = '0.0.1' schema = textwrap.dedent("""\ @@ -291,15 +281,19 @@ def create( dataset_id: str, table_id: str, ) -> 'BQTable': + # fmt: off """Create a BQTable artifact instance. Args: name: The artifact name. project_id: The ID of the project containing this table. dataset_id: The ID of the dataset containing this table. - table_id: The ID of the table. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/TableReference + table_id: The ID of the table. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/TableReference + + Returns: + BQTable instance. """ + # fmt: on return cls( name=name, uri=f'https://www.googleapis.com/bigquery/v2/projects/{project_id}/datasets/{dataset_id}/tables/{table_id}', @@ -312,7 +306,7 @@ def create( class UnmanagedContainerModel(dsl.Artifact): - """An artifact representing a Vertex AI `unmanaged container model `_.""" + """An artifact representing a Vertex AI [unmanaged container model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ModelContainerSpec).""" schema_title = 'google.UnmanagedContainerModel' schema_version = '0.0.1' schema = textwrap.dedent("""\ @@ -365,21 +359,20 @@ class UnmanagedContainerModel(dsl.Artifact): @classmethod def create( cls, - predict_schemata: Dict, - container_spec: Dict, + predict_schemata: Dict[str, str], + container_spec: Dict[str, Any], ) -> 'UnmanagedContainerModel': + # fmt: off """Create a UnmanagedContainerModel artifact instance. Args: - predict_schemata: Contains the schemata used in Model's predictions and - explanations via PredictionService.Predict, PredictionService.Explain - and BatchPredictionJob. For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/PredictSchemata - container_spec: Specification of a container for serving predictions. Some - fields in this message correspond to fields in the Kubernetes Container - v1 core specification. For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ModelContainerSpec + predict_schemata: Contains the schemata used in Model's predictions and explanations via PredictionService.Predict, PredictionService.Explain and BatchPredictionJob. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/PredictSchemata + container_spec: Specification of a container for serving predictions. Some fields in this message correspond to fields in the Kubernetes Container v1 core specification. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ModelContainerSpec + + Returns: + UnmanagedContainerModel instance. """ + # fmt: on return cls( metadata={ 'predictSchemata': predict_schemata, @@ -389,7 +382,7 @@ def create( class ClassificationMetrics(dsl.Artifact): - """An artifact representing evaluation `classification metrics `_.""" + """An artifact representing evaluation [classification metrics](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/evaluate-model#classification_1).""" schema_title = 'google.ClassificationMetrics' schema_version = '0.0.1' @@ -399,10 +392,7 @@ class ClassificationMetrics(dsl.Artifact): properties: aggregationType: type: string - enum: - - AGGREGATION_TYPE_UNSPECIFIED - - MACRO_AVERAGE - - MICRO_AVERAGE + enum: - AGGREGATION_TYPE_UNSPECIFIED - MACRO_AVERAGE - MICRO_AVERAGE aggregationThreshold: type: number format: float @@ -528,9 +518,11 @@ def create( au_roc: Optional[float] = None, log_loss: Optional[float] = None, ) -> 'ClassificationMetrics': + # fmt: off """Create a ClassificationMetrics artifact instance. Args: + name: The artifact name. recall: Recall (True Positive Rate) for the given confidence threshold. precision: Precision for the given confidence threshold. f1_score: The harmonic mean of recall and precision. @@ -538,7 +530,11 @@ def create( au_prc: The Area Under Precision-Recall Curve metric. au_roc: The Area Under Receiver Operating Characteristic curve metric. log_loss: The Log Loss metric. + + Returns: + ClassificationMetrics instance. """ + # fmt: on metadata = {} if recall is not None: metadata['recall'] = recall @@ -561,7 +557,7 @@ def create( class RegressionMetrics(dsl.Artifact): - """An artifact representing evaluation `regression metrics `_.""" + """An artifact representing evaluation [regression metrics](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/evaluate-model#regression_1).""" schema_title = 'google.RegressionMetrics' schema_version = '0.0.1' @@ -595,16 +591,21 @@ def create( r_squared: Optional[float] = None, root_mean_squared_log_error: Optional[float] = None, ) -> 'RegressionMetrics': + # fmt: off """Create a RegressionMetrics artifact instance. Args: + name: The artifact name. root_mean_squared_error: Root Mean Squared Error (RMSE). mean_absolute_error: Mean Absolute Error (MAE). mean_absolute_percentage_error: Mean absolute percentage error. - r_squared: Coefficient of determination as Pearson correlation - coefficient. + r_squared: Coefficient of determination as Pearson correlation coefficient. root_mean_squared_log_error: Root mean squared log error. + + Returns: + RegressionMetrics instance. """ + # fmt: on metadata = {} if root_mean_squared_error is not None: metadata['rootMeanSquaredError'] = root_mean_squared_error @@ -623,7 +624,7 @@ def create( class ForecastingMetrics(dsl.Artifact): - """An artifact representing evaluation `forecasting metrics `_.""" + """An artifact representing evaluation [forecasting metrics](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/evaluate-model#metrics).""" schema_title = 'google.ForecastingMetrics' schema_version = '0.0.1' @@ -683,24 +684,24 @@ def create( root_mean_squared_percentage_error: Optional[float] = None, symmetric_mean_absolute_percentage_error: Optional[float] = None, ) -> 'ForecastingMetrics': + # fmt: off """Create a ForecastingMetrics artifact instance. Args: + name: The artifact name. root_mean_squared_error: Root Mean Squared Error (RMSE). mean_absolute_error: Mean Absolute Error (MAE). mean_absolute_percentage_error: Mean absolute percentage error. - r_squared: Coefficient of determination as Pearson correlation - coefficient. + r_squared: Coefficient of determination as Pearson correlation coefficient. root_mean_squared_log_error: Root mean squared log error. - weighted_absolute_percentage_error: Weighted Absolute Percentage Error. - Does not use weights, this is just what the metric is called. Undefined - if actual values sum to zero. Will be very large if actual values sum to - a very small number. - root_mean_squared_percentage_error: Root Mean Square Percentage Error. - Square root of MSPE. Undefined/imaginary when MSPE is negative. - symmetric_mean_absolute_percentage_error: Symmetric Mean Absolute - Percentage Error. + weighted_absolute_percentage_error: Weighted Absolute Percentage Error. Does not use weights, this is just what the metric is called. Undefined if actual values sum to zero. Will be very large if actual values sum to a very small number. + root_mean_squared_percentage_error: Root Mean Square Percentage Error. Square root of MSPE. Undefined/imaginary when MSPE is negative. + symmetric_mean_absolute_percentage_error: Symmetric Mean Absolute Percentage Error. + + Returns: + ForecastingMetrics instance. """ + # fmt: on metadata = {} if root_mean_squared_error is not None: metadata['rootMeanSquaredError'] = root_mean_squared_error diff --git a/components/google-cloud/google_cloud_pipeline_components/utils.py b/components/google-cloud/google_cloud_pipeline_components/utils.py index c8939399a3..3ff0ba53d2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/utils.py +++ b/components/google-cloud/google_cloud_pipeline_components/utils.py @@ -16,16 +16,17 @@ import copy import json import re -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional from google_cloud_pipeline_components import _image from kfp import components from kfp import dsl +# do not follow this pattern! +# we should not depend on non-public modules of the KFP SDK! from kfp.components import placeholders from google.protobuf import json_format - # note: this is a slight dependency on KFP SDK implementation details # other code should not similarly depend on the stability of kfp.placeholders DOCS_INTEGRATED_OUTPUT_RENAMING_PREFIX = "output__" @@ -44,7 +45,7 @@ def build_serverless_customjob_container_spec( project: Project to run the job in. location: Location to run the job in. custom_job_payload: Payload to pass to the custom job. This dictionary is - serialized and passed as the custom job ``--payload``. + serialized and passed as the custom job `--payload`. gcp_resources: GCP resources that can be used to track the job. Returns: @@ -143,12 +144,19 @@ def unquote_nonstring_placeholders( def gcpc_output_name_converter( new_name: str, original_name: Optional[str] = None, -): +) -> Callable[["BaseComponent"], "BaseComponent"]: # pytype: disable=name-error """Replace the output with original_name with a new_name in a component decorated with an @dsl.container_component decorator. Enables authoring components that have an input and output with the same key/name. + Args: + new_name: The new name for the output. + original_name: The original name of the output. + + Returns: + A decorator that takes modifies a component in place. + Example usage: @utils.gcpc_output_name_converter('output__gcp_resources', 'gcp_resources') @@ -254,8 +262,8 @@ def replace_output_name_in_dag_outputs( ) def replace_output_name_in_executor( - command: list, - args: list, + command: List[str], + args: List[str], original_name: str, new_name: str, ): diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/__init__.py new file mode 100644 index 0000000000..6dbcd85caf --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GA AutoML forecasting components.""" + +from google_cloud_pipeline_components.v1.automl.forecasting.prophet_trainer import prophet_trainer as ProphetTrainerOp + +__all__ = [ + 'ProphetTrainerOp', +] diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml new file mode 100644 index 0000000000..820de13388 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_predict_pipeline.yaml @@ -0,0 +1,1162 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-bqml-arima-prediction +# Description: Forecasts using a BQML ARIMA_PLUS model. +# Inputs: +# bigquery_destination_uri: str [Default: ''] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# encryption_spec_key_name: str [Default: ''] +# generate_explanation: bool [Default: False] +# location: str +# model_name: str +# project: str +components: + comp-bigquery-create-dataset: + executorLabel: exec-bigquery-create-dataset + inputDefinitions: + parameters: + dataset: + parameterType: STRING + exists_ok: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + comp-bigquery-create-dataset-2: + executorLabel: exec-bigquery-create-dataset-2 + inputDefinitions: + parameters: + dataset: + parameterType: STRING + exists_ok: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + comp-bigquery-delete-dataset-with-prefix: + executorLabel: exec-bigquery-delete-dataset-with-prefix + inputDefinitions: + parameters: + dataset_prefix: + parameterType: STRING + delete_contents: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + project: + parameterType: STRING + comp-bigquery-query-job: + executorLabel: exec-bigquery-query-job + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-build-job-configuration-query: + executorLabel: exec-build-job-configuration-query + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-exit-handler-1: + dag: + tasks: + bigquery-create-dataset: + cachingOptions: {} + componentRef: + name: comp-bigquery-create-dataset + dependentTasks: + - get-table-location + - validate-inputs + inputs: + parameters: + dataset: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: create-tmp-dataset + bigquery-create-dataset-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-create-dataset-2 + dependentTasks: + - get-table-location + - maybe-replace-with-default + - validate-inputs + inputs: + parameters: + dataset: + taskOutputParameter: + outputParameterKey: Output + producerTask: maybe-replace-with-default + exists_ok: + runtimeValue: + constant: true + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: create-prediction-dataset + bigquery-query-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job + dependentTasks: + - build-job-configuration-query + - get-first-valid + - get-model-metadata + - get-table-location + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + pipelinechannel--get-first-valid-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-first-valid + pipelinechannel--get-model-metadata-forecast_horizon: + taskOutputParameter: + outputParameterKey: forecast_horizon + producerTask: get-model-metadata + pipelinechannel--get-model-metadata-target_column: + taskOutputParameter: + outputParameterKey: target_column + producerTask: get-model-metadata + pipelinechannel--get-model-metadata-time_column: + taskOutputParameter: + outputParameterKey: time_column + producerTask: get-model-metadata + pipelinechannel--get-model-metadata-time_series_identifier_column: + taskOutputParameter: + outputParameterKey: time_series_identifier_column + producerTask: get-model-metadata + pipelinechannel--model_name: + componentInputParameter: pipelinechannel--model_name + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n SELECT\n target.*,\n STRUCT(prediction.time_series_adjusted_data\ + \ AS value)\n AS predicted_{{$.inputs.parameters['pipelinechannel--get-model-metadata-target_column']}},\n\ + \ prediction.* EXCEPT (\n {{$.inputs.parameters['pipelinechannel--get-model-metadata-time_series_identifier_column']}},\n\ + \ time_series_timestamp,\n time_series_adjusted_data\n\ + \ ),\n FROM\n ML.EXPLAIN_FORECAST(\n \ + \ MODEL `{{$.inputs.parameters['pipelinechannel--model_name']}}`,\n\ + \ STRUCT({{$.inputs.parameters['pipelinechannel--get-model-metadata-forecast_horizon']}}\ + \ AS horizon)) AS prediction\n RIGHT JOIN `{{$.inputs.parameters['pipelinechannel--get-first-valid-Output']}}`\ + \ AS target\n ON\n CAST(target.{{$.inputs.parameters['pipelinechannel--get-model-metadata-time_series_identifier_column']}}\ + \ AS STRING)\n = CAST(prediction.{{$.inputs.parameters['pipelinechannel--get-model-metadata-time_series_identifier_column']}}\ + \ AS STRING)\n AND TIMESTAMP(target.{{$.inputs.parameters['pipelinechannel--get-model-metadata-time_column']}})\ + \ = prediction.time_series_timestamp\n WHERE target.{{$.inputs.parameters['pipelinechannel--get-model-metadata-target_column']}}\ + \ IS NULL\n " + taskInfo: + name: predictions-table + build-job-configuration-query: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query + dependentTasks: + - bigquery-create-dataset-2 + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-dataset_id'']}}' + pipelinechannel--bigquery-create-dataset-2-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset-2 + pipelinechannel--bigquery-create-dataset-2-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset-2 + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-project_id'']}}' + table_id: + runtimeValue: + constant: predictions_{{$.pipeline_job_uuid}} + taskInfo: + name: build-job-configuration-query + get-first-valid: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-first-valid + dependentTasks: + - load-table-from-uri + inputs: + parameters: + pipelinechannel--data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + pipelinechannel--load-table-from-uri-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: load-table-from-uri + values: + runtimeValue: + constant: '["{{$.inputs.parameters[''pipelinechannel--data_source_bigquery_table_path'']}}", + "{{$.inputs.parameters[''pipelinechannel--load-table-from-uri-Output'']}}"]' + taskInfo: + name: get-first-valid + get-model-metadata: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-model-metadata + dependentTasks: + - get-table-location + - validate-inputs + inputs: + parameters: + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + model: + componentInputParameter: pipelinechannel--model_name + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-model-metadata + get-table-location: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-table-location + inputs: + parameters: + default_location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + table: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + taskInfo: + name: get-table-location + load-table-from-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-load-table-from-uri + dependentTasks: + - bigquery-create-dataset + - get-table-location + inputs: + parameters: + destination: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}.{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}.csv_export' + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + project: + componentInputParameter: pipelinechannel--project + source_format: + runtimeValue: + constant: CSV + source_uris: + componentInputParameter: pipelinechannel--data_source_csv_filenames + taskInfo: + name: load-table-from-uri + maybe-replace-with-default: + cachingOptions: + enableCache: true + componentRef: + name: comp-maybe-replace-with-default + inputs: + parameters: + default: + runtimeValue: + constant: prediction_{{$.pipeline_job_uuid}} + value: + componentInputParameter: pipelinechannel--bigquery_destination_uri + taskInfo: + name: maybe-replace-with-default + validate-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-validate-inputs + inputs: + parameters: + bigquery_destination_uri: + componentInputParameter: pipelinechannel--bigquery_destination_uri + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--data_source_csv_filenames + source_model_uri: + componentInputParameter: pipelinechannel--model_name + taskInfo: + name: validate-inputs + inputDefinitions: + parameters: + pipelinechannel--bigquery_destination_uri: + parameterType: STRING + pipelinechannel--data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--data_source_csv_filenames: + parameterType: STRING + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_name: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + comp-get-first-valid: + executorLabel: exec-get-first-valid + inputDefinitions: + parameters: + values: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-model-metadata: + executorLabel: exec-get-model-metadata + inputDefinitions: + parameters: + location: + parameterType: STRING + model: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + forecast_horizon: + parameterType: NUMBER_INTEGER + target_column: + parameterType: STRING + time_column: + parameterType: STRING + time_series_identifier_column: + parameterType: STRING + comp-get-table-location: + executorLabel: exec-get-table-location + inputDefinitions: + parameters: + default_location: + defaultValue: '' + description: Location to return if no table was given. + isOptional: true + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + table: + description: The BigQuery table to get a location for. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-load-table-from-uri: + executorLabel: exec-load-table-from-uri + inputDefinitions: + parameters: + destination: + description: Table into which data is to be loaded. + parameterType: STRING + location: + description: The GCP region. + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + source_format: + defaultValue: CSV + description: 'The file format for the files being imported. Only CSV is + + supported.' + isOptional: true + parameterType: STRING + source_uris: + description: 'URIs of data files to be loaded; in format + + gs:///.' + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-maybe-replace-with-default: + executorLabel: exec-maybe-replace-with-default + inputDefinitions: + parameters: + default: + defaultValue: '' + isOptional: true + parameterType: STRING + value: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-validate-inputs: + executorLabel: exec-validate-inputs + inputDefinitions: + parameters: + bigquery_destination_uri: + isOptional: true + parameterType: STRING + data_granularity_unit: + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + isOptional: true + parameterType: STRING + data_source_csv_filenames: + isOptional: true + parameterType: STRING + optimization_objective: + isOptional: true + parameterType: STRING + predefined_split_key: + isOptional: true + parameterType: STRING + source_model_uri: + isOptional: true + parameterType: STRING + target_column: + isOptional: true + parameterType: STRING + test_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + isOptional: true + parameterType: STRING + time_series_identifier_column: + isOptional: true + parameterType: STRING + timestamp_split_key: + isOptional: true + parameterType: STRING + training_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + window_column: + isOptional: true + parameterType: STRING + window_max_count: + isOptional: true + parameterType: NUMBER_INTEGER + window_stride_length: + isOptional: true + parameterType: NUMBER_INTEGER +deploymentSpec: + executors: + exec-bigquery-create-dataset: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_create_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_create_dataset(\n project: str,\n location: str,\n\ + \ dataset: str,\n exists_ok: bool = False,\n) -> NamedTuple('Outputs',\ + \ [('project_id', str), ('dataset_id', str)]):\n \"\"\"Creates a BigQuery\ + \ dataset.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n\n from google.cloud import bigquery\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n ref\ + \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ + \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ + \ ref.project, ref.dataset_id)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-create-dataset-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_create_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_create_dataset(\n project: str,\n location: str,\n\ + \ dataset: str,\n exists_ok: bool = False,\n) -> NamedTuple('Outputs',\ + \ [('project_id', str), ('dataset_id', str)]):\n \"\"\"Creates a BigQuery\ + \ dataset.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n\n from google.cloud import bigquery\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n ref\ + \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ + \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ + \ ref.project, ref.dataset_id)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-delete-dataset-with-prefix: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_delete_dataset_with_prefix + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_delete_dataset_with_prefix(\n project: str,\n \ + \ dataset_prefix: str,\n delete_contents: bool = False,\n) -> None:\n\ + \ \"\"\"Deletes all BigQuery datasets matching the given prefix.\"\"\"\n\ + \ # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project)\n for dataset in client.list_datasets(project=project):\n\ + \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ + \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-query-job: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-build-job-configuration-query: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-first-valid: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_first_valid + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_first_valid(values: str) -> str:\n \"\"\"Returns the first\ + \ truthy value from the given serialized JSON list.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n for value in json.loads(values):\n if value:\n return value\n\ + \ raise ValueError('No valid values.')\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-model-metadata: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_model_metadata + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_model_metadata(\n project: str,\n location: str,\n\ + \ model: str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('time_column',\ + \ str),\n ('time_series_identifier_column', str),\n ('target_column',\ + \ str),\n ('forecast_horizon', int),\n ],\n):\n \"\"\"Retrieves\ + \ training options for a BQML model.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n\n from google.cloud import bigquery\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n options\ + \ = client.get_model(model).training_runs[0].training_options\n return\ + \ collections.namedtuple(\n 'Outputs', [\n 'time_column',\n\ + \ 'time_series_identifier_column',\n 'target_column',\n\ + \ 'forecast_horizon',\n ],\n )(\n options.time_series_timestamp_column,\n\ + \ options.time_series_id_column,\n options.time_series_data_column,\n\ + \ options.horizon,\n )\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-table-location: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_table_location + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_table_location(\n project: str,\n table: Optional[str],\n\ + \ default_location: str = '',\n) -> str:\n \"\"\"Returns the region\ + \ the given table belongs to.\n\n Args:\n project: The GCP project.\n\ + \ table: The BigQuery table to get a location for.\n default_location:\ + \ Location to return if no table was given.\n\n Returns:\n A GCP region\ + \ or multi-region.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n if not table:\n return default_location\n\n client = bigquery.Client(project=project)\n\ + \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ + \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ + \ return client.get_table(table).location\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-load-table-from-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - load_table_from_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef load_table_from_uri(\n project: str,\n location: str,\n\ + \ source_uris: str,\n destination: str,\n source_format: str =\ + \ 'CSV',\n) -> str:\n \"\"\"Creates a table from a list of URIs.\n\n Args:\n\ + \ project: The GCP project.\n location: The GCP region.\n source_uris:\ + \ URIs of data files to be loaded; in format\n gs:///.\n\ + \ destination: Table into which data is to be loaded.\n source_format:\ + \ The file format for the files being imported. Only CSV is\n supported.\n\ + \n Returns:\n The destination table containing imported data.\n \"\"\ + \"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n if not source_uris:\n return ''\n\n csv_list = [filename.strip()\ + \ for filename in source_uris.split(',')]\n client = bigquery.Client(project=project,\ + \ location=location)\n job_config = bigquery.LoadJobConfig(\n autodetect=True,\ + \ source_format=source_format)\n client.load_table_from_uri(\n source_uris=csv_list,\n\ + \ destination=destination,\n project=project,\n location=location,\n\ + \ job_config=job_config).result()\n return destination\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-maybe-replace-with-default: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - maybe_replace_with_default + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef maybe_replace_with_default(value: str, default: str = '') ->\ + \ str:\n \"\"\"Replaces string with another value if it is a dash.\"\"\"\ + \n return default if not value else value\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-validate-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - validate_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef validate_inputs(\n time_column: Optional[str] = None,\n \ + \ time_series_identifier_column: Optional[str] = None,\n target_column:\ + \ Optional[str] = None,\n data_source_bigquery_table_path: Optional[str]\ + \ = None,\n training_fraction: Optional[float] = None,\n validation_fraction:\ + \ Optional[float] = None,\n test_fraction: Optional[float] = None,\n\ + \ predefined_split_key: Optional[str] = None,\n timestamp_split_key:\ + \ Optional[str] = None,\n data_source_csv_filenames: Optional[str] =\ + \ None,\n source_model_uri: Optional[str] = None,\n bigquery_destination_uri:\ + \ Optional[str] = None,\n window_column: Optional[str] = None,\n window_stride_length:\ + \ Optional[int] = None,\n window_max_count: Optional[int] = None,\n \ + \ optimization_objective: Optional[str] = None,\n data_granularity_unit:\ + \ Optional[str] = None,\n) -> None:\n \"\"\"Checks training pipeline input\ + \ parameters are valid.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import re\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n project_pattern = r'([a-z0-9.-]+:)?[a-z][a-z0-9-_]{4,28}[a-z0-9]'\n\ + \ dataset_pattern = r'[a-zA-Z0-9_]+'\n table_pattern = r'[^\\.\\:`]+'\n\ + \ dataset_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}')\n\ + \ table_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}[.:]{table_pattern}')\n\ + \n # Validate BigQuery column and dataset names.\n bigquery_column_parameters\ + \ = [\n time_column,\n time_series_identifier_column,\n target_column,\n\ + \ ]\n column_pattern = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]{1,300}')\n \ + \ for column in bigquery_column_parameters:\n if column and not column_pattern.fullmatch(column):\n\ + \ raise ValueError(f'Invalid column name: {column}.')\n if (bigquery_destination_uri\ + \ and\n not dataset_uri_pattern.fullmatch(bigquery_destination_uri)):\n\ + \ raise ValueError(\n f'Invalid BigQuery dataset URI: {bigquery_destination_uri}.')\n\ + \ if (source_model_uri and not table_uri_pattern.fullmatch(source_model_uri)):\n\ + \ raise ValueError(f'Invalid BigQuery table URI: {source_model_uri}.')\n\ + \n # Validate data source.\n data_source_count = sum([bool(source) for\ + \ source in [\n data_source_bigquery_table_path, data_source_csv_filenames]])\n\ + \ if data_source_count > 1:\n raise ValueError(f'Expected 1 data source,\ + \ found {data_source_count}.')\n if (data_source_bigquery_table_path\n\ + \ and not table_uri_pattern.fullmatch(data_source_bigquery_table_path)):\n\ + \ raise ValueError(\n f'Invalid BigQuery table URI: {data_source_bigquery_table_path}.')\n\ + \ gcs_path_pattern = re.compile(r'gs:\\/\\/(.+)\\/([^\\/]+)')\n if data_source_csv_filenames:\n\ + \ csv_list = [filename.strip()\n for filename in data_source_csv_filenames.split(',')]\n\ + \ for gcs_path in csv_list:\n if not gcs_path_pattern.fullmatch(gcs_path):\n\ + \ raise ValueError(f'Invalid path to CSV stored in GCS: {gcs_path}.')\n\ + \n # Validate split spec.\n fraction_splits = [\n training_fraction,\n\ + \ validation_fraction,\n test_fraction,\n ]\n fraction_splits\ + \ = [None if fraction == -1 else fraction\n for fraction\ + \ in fraction_splits]\n split_count = sum([\n bool(source)\n \ + \ for source in [predefined_split_key,\n any(fraction_splits)]\n\ + \ ])\n if split_count > 1:\n raise ValueError(f'Expected 1 split type,\ + \ found {split_count}.')\n if (predefined_split_key and\n not column_pattern.fullmatch(predefined_split_key)):\n\ + \ raise ValueError(f'Invalid column name: {predefined_split_key}.')\n\ + \ if any(fraction_splits):\n if not all(fraction_splits):\n raise\ + \ ValueError(\n f'All fractions must be non-zero. Got: {fraction_splits}.')\n\ + \ if sum(fraction_splits) != 1:\n raise ValueError(\n f'Fraction\ + \ splits must sum to 1. Got: {sum(fraction_splits)}.')\n if (timestamp_split_key\ + \ and\n not column_pattern.fullmatch(timestamp_split_key)):\n raise\ + \ ValueError(f'Invalid column name: {timestamp_split_key}.')\n if timestamp_split_key\ + \ and not all(fraction_splits):\n raise ValueError('All fractions must\ + \ be non-zero for timestamp split.')\n\n # Validate window config.\n if\ + \ window_stride_length == -1:\n window_stride_length = None\n if window_max_count\ + \ == -1:\n window_max_count = None\n window_configs = [window_column,\ + \ window_stride_length, window_max_count]\n window_config_count = sum([bool(config)\ + \ for config in window_configs])\n if window_config_count > 1:\n raise\ + \ ValueError(f'Expected 1 window config, found {window_config_count}.')\n\ + \ if window_column and not column_pattern.fullmatch(window_column):\n \ + \ raise ValueError(f'Invalid column name: {window_column}.')\n if window_stride_length\ + \ and (window_stride_length < 1 or\n window_stride_length\ + \ > 1000):\n raise ValueError('Stride must be between 1 and 1000. Got:\ + \ '\n f'{window_stride_length}.')\n if window_max_count\ + \ and (window_max_count < 1000 or\n window_max_count\ + \ > int(1e8)):\n raise ValueError('Max count must be between 1000 and\ + \ 100000000. Got: '\n f'{window_max_count}.')\n\n #\ + \ Validate eval metric.\n valid_optimization_objectives = ['rmse', 'mae',\ + \ 'rmsle']\n if optimization_objective:\n if optimization_objective\ + \ not in valid_optimization_objectives:\n raise ValueError(\n \ + \ 'Optimization objective should be one of the following: '\n \ + \ f'{valid_optimization_objectives}, got: {optimization_objective}.')\n\ + \n # Validate data granularity unit.\n valid_data_granularity_units =\ + \ [\n 'minute', 'hour', 'day', 'week', 'month', 'year']\n if data_granularity_unit:\n\ + \ if data_granularity_unit not in valid_data_granularity_units:\n \ + \ raise ValueError(\n 'Granularity unit should be one of the\ + \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 +pipelineInfo: + description: Forecasts using a BQML ARIMA_PLUS model. + name: automl-tabular-bqml-arima-prediction +root: + dag: + tasks: + bigquery-delete-dataset-with-prefix: + cachingOptions: {} + componentRef: + name: comp-bigquery-delete-dataset-with-prefix + dependentTasks: + - exit-handler-1 + inputs: + parameters: + dataset_prefix: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + delete_contents: + runtimeValue: + constant: true + project: + componentInputParameter: project + taskInfo: + name: delete-tmp-dataset + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + inputs: + parameters: + pipelinechannel--bigquery_destination_uri: + componentInputParameter: bigquery_destination_uri + pipelinechannel--data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + pipelinechannel--data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--model_name: + componentInputParameter: model_name + pipelinechannel--project: + componentInputParameter: project + taskInfo: + name: exit-handler-1 + inputDefinitions: + parameters: + bigquery_destination_uri: + defaultValue: '' + description: 'URI of the desired destination dataset. If not + + specified, a resource will be created under a new dataset in the project.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + generate_explanation: + defaultValue: false + description: 'Generate explanation along with the batch prediction + + results. This will cause the batch prediction output to include + + explanations.' + isOptional: true + parameterType: BOOLEAN + location: + description: The GCP region for Vertex AI. + parameterType: STRING + model_name: + description: ARIMA_PLUS BQML model URI. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml new file mode 100644 index 0000000000..64e19d3bab --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/bqml_arima_train_pipeline.yaml @@ -0,0 +1,5196 @@ +# PIPELINE DEFINITION +# Name: automl-tabular-bqml-arima-train +# Description: Trains a BQML ARIMA_PLUS model. +# Inputs: +# bigquery_destination_uri: str [Default: ''] +# data_granularity_unit: str +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# encryption_spec_key_name: str [Default: ''] +# forecast_horizon: int +# location: str +# max_order: int [Default: 5.0] +# override_destination: bool [Default: False] +# predefined_split_key: str [Default: ''] +# project: str +# root_dir: str +# run_evaluation: bool [Default: True] +# target_column: str +# test_fraction: float [Default: -1.0] +# time_column: str +# time_series_identifier_column: str +# timestamp_split_key: str [Default: ''] +# training_fraction: float [Default: -1.0] +# validation_fraction: float [Default: -1.0] +# window_column: str [Default: ''] +# window_max_count: int [Default: -1.0] +# window_stride_length: int [Default: -1.0] +# Outputs: +# create-metrics-artifact-evaluation_metrics: system.Metrics +components: + comp-bigquery-create-dataset: + executorLabel: exec-bigquery-create-dataset + inputDefinitions: + parameters: + dataset: + parameterType: STRING + exists_ok: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + comp-bigquery-create-dataset-2: + executorLabel: exec-bigquery-create-dataset-2 + inputDefinitions: + parameters: + dataset: + parameterType: STRING + exists_ok: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + comp-bigquery-create-model-job: + executorLabel: exec-bigquery-create-model-job + inputDefinitions: + parameters: + job_configuration_query: + defaultValue: {} + description: 'A json formatted string describing the rest of the job configuration. + + For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: "The labels associated with this job. You can\nuse these to\ + \ organize and group your jobs. Label keys and values can\nbe no longer\ + \ than 63 characters, can only containlowercase letters,\nnumeric characters,\ + \ underscores and dashes. International characters\nare allowed. Label\ + \ values are optional. Label keys must start with a\nletter and each label\ + \ in the list must have a different key.\n Example: { \"name\": \"wrench\"\ + , \"mass\": \"1.3kg\", \"count\": \"3\" }." + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location of the job to create the BigQuery model. If not set, + default to + + `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run BigQuery model creation job. Defaults to the + project in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'Query parameters for standard SQL queries. + + If query_parameters are both specified in here and in + + job_configuration_query, the value in here will override the other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.BQMLModel + schemaVersion: 0.0.1 + description: Describes the model which is created. + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bigquery-delete-dataset-with-prefix: + executorLabel: exec-bigquery-delete-dataset-with-prefix + inputDefinitions: + parameters: + dataset_prefix: + parameterType: STRING + delete_contents: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + project: + parameterType: STRING + comp-bigquery-list-rows: + executorLabel: exec-bigquery-list-rows + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: A google.BQTable artifact. + parameters: + location: + description: The GCP region. + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-bigquery-list-rows-2: + executorLabel: exec-bigquery-list-rows-2 + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: A google.BQTable artifact. + parameters: + location: + description: The GCP region. + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-bigquery-query-job: + executorLabel: exec-bigquery-query-job + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bigquery-query-job-2: + executorLabel: exec-bigquery-query-job-2 + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bigquery-query-job-3: + executorLabel: exec-bigquery-query-job-3 + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bigquery-query-job-4: + executorLabel: exec-bigquery-query-job-4 + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bigquery-query-job-5: + executorLabel: exec-bigquery-query-job-5 + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-build-job-configuration-query: + executorLabel: exec-build-job-configuration-query + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-build-job-configuration-query-2: + executorLabel: exec-build-job-configuration-query-2 + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-build-job-configuration-query-3: + executorLabel: exec-build-job-configuration-query-3 + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-build-job-configuration-query-4: + executorLabel: exec-build-job-configuration-query-4 + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-build-job-configuration-query-5: + executorLabel: exec-build-job-configuration-query-5 + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-build-job-configuration-query-6: + executorLabel: exec-build-job-configuration-query-6 + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-build-serialized-query-parameters: + executorLabel: exec-build-serialized-query-parameters + inputDefinitions: + parameters: + data_granularity_unit: + description: 'The data granularity unit. Accepted values are: + + minute, hour, day, week, month, year.' + isOptional: true + parameterType: STRING + forecast_horizon: + description: 'The number of time periods into the future for which + + forecasts will be created. Future periods start after the latest timestamp + + for each time series.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon_off_by_one: + defaultValue: false + description: 'If True, subtract 1 from the forecast horizon + + in the query parameters.' + isOptional: true + parameterType: BOOLEAN + max_order: + description: 'Integer between 1 and 5 representing the size of the parameter + + search space for ARIMA_PLUS. 5 would result in the highest accuracy model, + + but also the longest training runtime.' + isOptional: true + parameterType: NUMBER_INTEGER + splits: + description: Dataset splits to be used to train the model. + isOptional: true + parameterType: LIST + window: + description: 'Dict containing information about the forecast window the + model + + should have. If no window is provided, the window will start after the + + latest period in the available data.' + isOptional: true + parameterType: STRUCT + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-build-serialized-query-parameters-2: + executorLabel: exec-build-serialized-query-parameters-2 + inputDefinitions: + parameters: + data_granularity_unit: + description: 'The data granularity unit. Accepted values are: + + minute, hour, day, week, month, year.' + isOptional: true + parameterType: STRING + forecast_horizon: + description: 'The number of time periods into the future for which + + forecasts will be created. Future periods start after the latest timestamp + + for each time series.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon_off_by_one: + defaultValue: false + description: 'If True, subtract 1 from the forecast horizon + + in the query parameters.' + isOptional: true + parameterType: BOOLEAN + max_order: + description: 'Integer between 1 and 5 representing the size of the parameter + + search space for ARIMA_PLUS. 5 would result in the highest accuracy model, + + but also the longest training runtime.' + isOptional: true + parameterType: NUMBER_INTEGER + splits: + description: Dataset splits to be used to train the model. + isOptional: true + parameterType: LIST + window: + description: 'Dict containing information about the forecast window the + model + + should have. If no window is provided, the window will start after the + + latest period in the available data.' + isOptional: true + parameterType: STRUCT + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-build-serialized-query-parameters-3: + executorLabel: exec-build-serialized-query-parameters-3 + inputDefinitions: + parameters: + data_granularity_unit: + description: 'The data granularity unit. Accepted values are: + + minute, hour, day, week, month, year.' + isOptional: true + parameterType: STRING + forecast_horizon: + description: 'The number of time periods into the future for which + + forecasts will be created. Future periods start after the latest timestamp + + for each time series.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon_off_by_one: + defaultValue: false + description: 'If True, subtract 1 from the forecast horizon + + in the query parameters.' + isOptional: true + parameterType: BOOLEAN + max_order: + description: 'Integer between 1 and 5 representing the size of the parameter + + search space for ARIMA_PLUS. 5 would result in the highest accuracy model, + + but also the longest training runtime.' + isOptional: true + parameterType: NUMBER_INTEGER + splits: + description: Dataset splits to be used to train the model. + isOptional: true + parameterType: LIST + window: + description: 'Dict containing information about the forecast window the + model + + should have. If no window is provided, the window will start after the + + latest period in the available data.' + isOptional: true + parameterType: STRUCT + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-cond: + executorLabel: exec-cond + inputDefinitions: + parameters: + false_str: + parameterType: STRING + predicate: + parameterType: BOOLEAN + true_str: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-condition-2: + dag: + outputs: + artifacts: + create-metrics-artifact-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: create-metrics-artifact + tasks: + bigquery-list-rows: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-list-rows + dependentTasks: + - bigquery-query-job + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: destination_table + producerTask: bigquery-query-job + parameters: + location: + componentInputParameter: pipelinechannel--get-table-location-Output + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: bigquery-list-rows + bigquery-list-rows-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-list-rows-2 + dependentTasks: + - bigquery-query-job-4 + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: destination_table + producerTask: bigquery-query-job-4 + parameters: + location: + componentInputParameter: pipelinechannel--get-table-location-Output + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: bigquery-list-rows-2 + bigquery-query-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job + dependentTasks: + - build-job-configuration-query + - build-serialized-query-parameters + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--bigquery-create-dataset-2-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-dataset_id + pipelinechannel--bigquery-create-dataset-2-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-project_id + pipelinechannel--data_granularity_unit: + componentInputParameter: pipelinechannel--data_granularity_unit + pipelinechannel--get-fte-suffix-Output: + componentInputParameter: pipelinechannel--get-fte-suffix-Output + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n WITH\n time_series_windows AS (\n \ + \ SELECT\n FIRST_VALUE({{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ OVER (horizon) AS start_time,\n COUNT(*) OVER (horizon)\ + \ AS count,\n FIRST_VALUE(window__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}})\ + \ OVER (horizon) AS window__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}},\n\ + \ FROM `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-dataset_id']}}.fte_time_series_output_{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}`\n\ + \ WHERE UPPER(split__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}})\ + \ IN UNNEST(@splits)\n WINDOW horizon AS (\n \ + \ PARTITION BY {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\n\ + \ ORDER BY {{$.inputs.parameters['pipelinechannel--time_column']}}\n\ + \ ROWS BETWEEN 0 PRECEDING AND @forecast_horizon FOLLOWING)\n\ + \ )\n SELECT\n start_time,\n TIMESTAMP(DATETIME_ADD(\n\ + \ DATETIME(start_time),\n INTERVAL @forecast_horizon\ + \ {{$.inputs.parameters['pipelinechannel--data_granularity_unit']}}\n\ + \ )) AS end_time,\n SUM(count) AS count,\n \ + \ ROW_NUMBER() OVER () AS window_number,\n FROM time_series_windows\n\ + \ WHERE window__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}\n\ + \ GROUP BY start_time\n " + query_parameters: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-serialized-query-parameters + taskInfo: + name: create-eval-windows-table + bigquery-query-job-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job-2 + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--bigquery-create-dataset-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-dataset_id + pipelinechannel--bigquery-create-dataset-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-project_id + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n CREATE TABLE `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-dataset_id']}}.metrics`\ + \ (\n predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}}\ + \ TIMESTAMP,\n MAE FLOAT64,\n MSE\ + \ FLOAT64,\n MAPE FLOAT64,\n prediction_count\ + \ INT64\n )\n " + taskInfo: + name: create-tmp-metrics-table + bigquery-query-job-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job-3 + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--bigquery-create-dataset-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-dataset_id + pipelinechannel--bigquery-create-dataset-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-project_id + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n CREATE TABLE `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-dataset_id']}}.evaluated_examples`\ + \ (\n {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\ + \ STRING,\n {{$.inputs.parameters['pipelinechannel--time_column']}}\ + \ TIMESTAMP,\n predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}}\ + \ TIMESTAMP,\n {{$.inputs.parameters['pipelinechannel--target_column']}}\ + \ FLOAT64,\n predicted_{{$.inputs.parameters['pipelinechannel--target_column']}}\ + \ STRUCT\n )\n " + taskInfo: + name: create-evaluated-examples-table + bigquery-query-job-4: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job-4 + dependentTasks: + - build-job-configuration-query-5 + - for-loop-3 + - table-to-uri + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query-5 + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--table-to-uri-uri: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n SELECT\n SUM(MAE * prediction_count) /\ + \ SUM(prediction_count) AS MAE,\n SQRT(SUM(MSE * prediction_count)\ + \ / SUM(prediction_count)) AS RMSE,\n SUM(MAPE * prediction_count)\ + \ / SUM(prediction_count) AS MAPE,\n FROM `{{$.inputs.parameters['pipelinechannel--table-to-uri-uri']}}`\n\ + \ " + taskInfo: + name: create-backtest-table + bigquery-query-job-5: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job-5 + dependentTasks: + - build-job-configuration-query-6 + - for-loop-3 + - table-to-uri-2 + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query-6 + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--table-to-uri-2-uri: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri-2 + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: SELECT * FROM `{{$.inputs.parameters['pipelinechannel--table-to-uri-2-uri']}}` + taskInfo: + name: export-evaluated-examples-table + build-job-configuration-query: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}' + pipelinechannel--bigquery-create-dataset-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-dataset_id + pipelinechannel--bigquery-create-dataset-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-project_id + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}' + table_id: + runtimeValue: + constant: windows + taskInfo: + name: build-job-configuration-query + build-job-configuration-query-5: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query-5 + dependentTasks: + - cond + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}' + pipelinechannel--bigquery-create-dataset-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-dataset_id + pipelinechannel--bigquery-create-dataset-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-project_id + pipelinechannel--cond-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: cond + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}' + table_id: + runtimeValue: + constant: final_metrics + write_disposition: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--cond-Output'']}}' + taskInfo: + name: build-job-configuration-query-5 + build-job-configuration-query-6: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query-6 + dependentTasks: + - cond + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-dataset_id'']}}' + pipelinechannel--bigquery-create-dataset-2-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-dataset_id + pipelinechannel--bigquery-create-dataset-2-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-project_id + pipelinechannel--cond-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: cond + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-project_id'']}}' + table_id: + runtimeValue: + constant: evaluated_examples + write_disposition: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--cond-Output'']}}' + taskInfo: + name: build-job-configuration-query-6 + build-serialized-query-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-serialized-query-parameters + inputs: + parameters: + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecast_horizon_off_by_one: + runtimeValue: + constant: true + splits: + runtimeValue: + constant: + - TEST + taskInfo: + name: build-serialized-query-parameters + cond: + cachingOptions: + enableCache: true + componentRef: + name: comp-cond + inputs: + parameters: + false_str: + runtimeValue: + constant: WRITE_EMPTY + predicate: + componentInputParameter: pipelinechannel--override_destination + true_str: + runtimeValue: + constant: WRITE_TRUNCATE + taskInfo: + name: cond + create-metrics-artifact: + cachingOptions: + enableCache: true + componentRef: + name: comp-create-metrics-artifact + dependentTasks: + - bigquery-list-rows-2 + inputs: + parameters: + metrics_rows: + taskOutputParameter: + outputParameterKey: Output + producerTask: bigquery-list-rows-2 + taskInfo: + name: create-metrics-artifact + for-loop-3: + componentRef: + name: comp-for-loop-3 + dependentTasks: + - bigquery-list-rows + - table-to-uri + - table-to-uri-2 + inputs: + parameters: + pipelinechannel--bigquery-create-dataset-2-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-dataset_id + pipelinechannel--bigquery-create-dataset-2-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-project_id + pipelinechannel--bigquery-create-dataset-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-dataset_id + pipelinechannel--bigquery-create-dataset-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-project_id + pipelinechannel--bigquery-list-rows-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bigquery-list-rows + pipelinechannel--data_granularity_unit: + componentInputParameter: pipelinechannel--data_granularity_unit + pipelinechannel--forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + pipelinechannel--get-fte-suffix-Output: + componentInputParameter: pipelinechannel--get-fte-suffix-Output + pipelinechannel--get-table-location-Output: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--max_order: + componentInputParameter: pipelinechannel--max_order + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--table-to-uri-2-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: table-to-uri-2 + pipelinechannel--table-to-uri-2-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: table-to-uri-2 + pipelinechannel--table-to-uri-2-table_id: + taskOutputParameter: + outputParameterKey: table_id + producerTask: table-to-uri-2 + pipelinechannel--table-to-uri-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: table-to-uri + pipelinechannel--table-to-uri-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: table-to-uri + pipelinechannel--table-to-uri-table_id: + taskOutputParameter: + outputParameterKey: table_id + producerTask: table-to-uri + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + iteratorPolicy: + parallelismLimit: 50 + parameterIterator: + itemInput: pipelinechannel--bigquery-list-rows-Output-loop-item + items: + inputParameter: pipelinechannel--bigquery-list-rows-Output + taskInfo: + name: for-loop-3 + table-to-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri + dependentTasks: + - bigquery-query-job-2 + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: destination_table + producerTask: bigquery-query-job-2 + taskInfo: + name: table-to-uri + table-to-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri-2 + dependentTasks: + - bigquery-query-job-3 + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: destination_table + producerTask: bigquery-query-job-3 + taskInfo: + name: table-to-uri-2 + inputDefinitions: + parameters: + pipelinechannel--bigquery-create-dataset-2-dataset_id: + parameterType: STRING + pipelinechannel--bigquery-create-dataset-2-project_id: + parameterType: STRING + pipelinechannel--bigquery-create-dataset-dataset_id: + parameterType: STRING + pipelinechannel--bigquery-create-dataset-project_id: + parameterType: STRING + pipelinechannel--data_granularity_unit: + parameterType: STRING + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--get-fte-suffix-Output: + parameterType: STRING + pipelinechannel--get-table-location-Output: + parameterType: STRING + pipelinechannel--max_order: + parameterType: NUMBER_INTEGER + pipelinechannel--override_destination: + parameterType: BOOLEAN + pipelinechannel--project: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_identifier_column: + parameterType: STRING + outputDefinitions: + artifacts: + create-metrics-artifact-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-create-metrics-artifact: + executorLabel: exec-create-metrics-artifact + inputDefinitions: + parameters: + metrics_rows: + parameterType: LIST + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + create-metrics-artifact-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: create-metrics-artifact-evaluation_metrics + producerSubtask: condition-2 + tasks: + bigquery-create-dataset: + cachingOptions: {} + componentRef: + name: comp-bigquery-create-dataset + dependentTasks: + - get-table-location + - validate-inputs + inputs: + parameters: + dataset: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: create-tmp-dataset + bigquery-create-dataset-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-create-dataset-2 + dependentTasks: + - get-table-location + - maybe-replace-with-default + - validate-inputs + inputs: + parameters: + dataset: + taskOutputParameter: + outputParameterKey: Output + producerTask: maybe-replace-with-default + exists_ok: + runtimeValue: + constant: true + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: create-export-dataset + bigquery-create-model-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-create-model-job + dependentTasks: + - bigquery-create-dataset-2 + - build-serialized-query-parameters-3 + - get-fte-suffix + - get-table-location + inputs: + parameters: + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + pipelinechannel--bigquery-create-dataset-2-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset-2 + pipelinechannel--bigquery-create-dataset-2-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset-2 + pipelinechannel--get-fte-suffix-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-fte-suffix + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n CREATE MODEL `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-dataset_id']}}.model_{{$.pipeline_job_uuid}}`\n\ + \ OPTIONS (\n model_type = 'ARIMA_PLUS',\n \ + \ time_series_timestamp_col = '{{$.inputs.parameters['pipelinechannel--time_column']}}',\n\ + \ time_series_id_col = '{{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}',\n\ + \ time_series_data_col = '{{$.inputs.parameters['pipelinechannel--target_column']}}',\n\ + \ horizon = @forecast_horizon,\n auto_arima\ + \ = True,\n auto_arima_max_order = @max_order,\n \ + \ data_frequency = @data_granularity_unit,\n holiday_region\ + \ = 'GLOBAL',\n clean_spikes_and_dips = True,\n \ + \ adjust_step_changes = True,\n decompose_time_series\ + \ = True\n ) AS\n SELECT\n {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}},\n\ + \ {{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ {{$.inputs.parameters['pipelinechannel--target_column']}},\n\ + \ FROM `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-dataset_id']}}.fte_time_series_output_{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}`\n\ + \ WHERE\n UPPER(split__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}})\ + \ IN UNNEST(@splits)\n AND TIMESTAMP({{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ < @start_time\n " + query_parameters: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-serialized-query-parameters-3 + taskInfo: + name: create-serving-model + build-serialized-query-parameters-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-serialized-query-parameters-3 + inputs: + parameters: + data_granularity_unit: + componentInputParameter: pipelinechannel--data_granularity_unit + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + max_order: + componentInputParameter: pipelinechannel--max_order + splits: + runtimeValue: + constant: + - TRAIN + - VALIDATE + - TEST + taskInfo: + name: build-serialized-query-parameters-3 + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - bigquery-create-dataset + - bigquery-create-dataset-2 + - get-fte-suffix + - get-table-location + inputs: + parameters: + pipelinechannel--bigquery-create-dataset-2-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset-2 + pipelinechannel--bigquery-create-dataset-2-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset-2 + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + pipelinechannel--data_granularity_unit: + componentInputParameter: pipelinechannel--data_granularity_unit + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + pipelinechannel--get-fte-suffix-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-fte-suffix + pipelinechannel--get-table-location-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + pipelinechannel--max_order: + componentInputParameter: pipelinechannel--max_order + pipelinechannel--override_destination: + componentInputParameter: pipelinechannel--override_destination + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + dependentTasks: + - bigquery-create-dataset-2 + - wrapped-in-list + inputs: + parameters: + autodetect_csv_schema: + runtimeValue: + constant: true + bigquery_staging_full_dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-project_id'']}}.{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-dataset_id'']}}' + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--data_source_csv_filenames + forecasting_apply_windowing: + runtimeValue: + constant: false + forecasting_context_window: + runtimeValue: + constant: 0.0 + forecasting_forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_predefined_window_column: + componentInputParameter: pipelinechannel--window_column + forecasting_time_column: + componentInputParameter: pipelinechannel--time_column + forecasting_time_series_identifier_columns: + taskOutputParameter: + outputParameterKey: Output + producerTask: wrapped-in-list + forecasting_window_max_count: + componentInputParameter: pipelinechannel--window_max_count + forecasting_window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--bigquery-create-dataset-2-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset-2 + pipelinechannel--bigquery-create-dataset-2-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset-2 + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + runtimeValue: + constant: time_series + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + runtimeValue: + constant: {} + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + taskInfo: + name: feature-transform-engine + get-fte-suffix: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-fte-suffix + dependentTasks: + - bigquery-create-dataset-2 + - feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-project_id'']}}.{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-2-dataset_id'']}}' + fte_table: + runtimeValue: + constant: fte_time_series_output + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--bigquery-create-dataset-2-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset-2 + pipelinechannel--bigquery-create-dataset-2-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset-2 + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-fte-suffix + get-table-location: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-table-location + inputs: + parameters: + default_location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + table: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + taskInfo: + name: get-table-location + maybe-replace-with-default: + cachingOptions: + enableCache: true + componentRef: + name: comp-maybe-replace-with-default + inputs: + parameters: + default: + runtimeValue: + constant: export_{{$.pipeline_job_uuid}} + value: + componentInputParameter: pipelinechannel--bigquery_destination_uri + taskInfo: + name: maybe-replace-with-default + validate-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-validate-inputs + inputs: + parameters: + bigquery_destination_uri: + componentInputParameter: pipelinechannel--bigquery_destination_uri + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--data_source_csv_filenames + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + time_column: + componentInputParameter: pipelinechannel--time_column + time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + window_column: + componentInputParameter: pipelinechannel--window_column + window_max_count: + componentInputParameter: pipelinechannel--window_max_count + window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + taskInfo: + name: validate-inputs + wrapped-in-list: + cachingOptions: + enableCache: true + componentRef: + name: comp-wrapped-in-list + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--time_series_identifier_column + taskInfo: + name: wrapped-in-list + inputDefinitions: + parameters: + pipelinechannel--bigquery_destination_uri: + parameterType: STRING + pipelinechannel--data_granularity_unit: + parameterType: STRING + pipelinechannel--data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--data_source_csv_filenames: + parameterType: STRING + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--max_order: + parameterType: NUMBER_INTEGER + pipelinechannel--override_destination: + parameterType: BOOLEAN + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_identifier_column: + parameterType: STRING + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--window_column: + parameterType: STRING + pipelinechannel--window_max_count: + parameterType: NUMBER_INTEGER + pipelinechannel--window_stride_length: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + create-metrics-artifact-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-for-loop-3: + dag: + tasks: + build-job-configuration-query-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query-2 + dependentTasks: + - get-window-query-priority + inputs: + parameters: + pipelinechannel--get-window-query-priority-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-window-query-priority + priority: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--get-window-query-priority-Output'']}}' + taskInfo: + name: build-job-configuration-query-2 + build-job-configuration-query-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query-3 + dependentTasks: + - get-window-query-priority + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-dataset_id'']}}' + pipelinechannel--get-window-query-priority-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-window-query-priority + pipelinechannel--table-to-uri-dataset_id: + componentInputParameter: pipelinechannel--table-to-uri-dataset_id + pipelinechannel--table-to-uri-project_id: + componentInputParameter: pipelinechannel--table-to-uri-project_id + pipelinechannel--table-to-uri-table_id: + componentInputParameter: pipelinechannel--table-to-uri-table_id + priority: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--get-window-query-priority-Output'']}}' + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-project_id'']}}' + table_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-table_id'']}}' + write_disposition: + runtimeValue: + constant: WRITE_APPEND + taskInfo: + name: build-job-configuration-query-3 + build-job-configuration-query-4: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query-4 + dependentTasks: + - get-window-query-priority + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-2-dataset_id'']}}' + pipelinechannel--get-window-query-priority-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-window-query-priority + pipelinechannel--table-to-uri-2-dataset_id: + componentInputParameter: pipelinechannel--table-to-uri-2-dataset_id + pipelinechannel--table-to-uri-2-project_id: + componentInputParameter: pipelinechannel--table-to-uri-2-project_id + pipelinechannel--table-to-uri-2-table_id: + componentInputParameter: pipelinechannel--table-to-uri-2-table_id + priority: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--get-window-query-priority-Output'']}}' + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-2-project_id'']}}' + table_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-2-table_id'']}}' + write_disposition: + runtimeValue: + constant: WRITE_APPEND + taskInfo: + name: build-job-configuration-query-4 + build-serialized-query-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-serialized-query-parameters-2 + inputs: + parameters: + data_granularity_unit: + componentInputParameter: pipelinechannel--data_granularity_unit + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + max_order: + componentInputParameter: pipelinechannel--max_order + splits: + runtimeValue: + constant: + - TRAIN + - VALIDATE + - TEST + window: + componentInputParameter: pipelinechannel--bigquery-list-rows-Output-loop-item + taskInfo: + name: build-serialized-query-parameters-2 + get-value: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-value + inputs: + parameters: + d: + componentInputParameter: pipelinechannel--bigquery-list-rows-Output-loop-item + key: + runtimeValue: + constant: window_number + taskInfo: + name: get_window_number + get-window-query-priority: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-window-query-priority + inputs: + parameters: + max_interactive: + runtimeValue: + constant: 50.0 + window: + componentInputParameter: pipelinechannel--bigquery-list-rows-Output-loop-item + taskInfo: + name: get-window-query-priority + query-with-retry: + cachingOptions: + enableCache: true + componentRef: + name: comp-query-with-retry + dependentTasks: + - build-job-configuration-query-2 + - build-serialized-query-parameters-2 + - get-value + inputs: + parameters: + destination_uri: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}.{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}.model_{{$.inputs.parameters[''pipelinechannel--get-value-Output'']}}' + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query-2 + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--bigquery-create-dataset-2-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-dataset_id + pipelinechannel--bigquery-create-dataset-2-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-project_id + pipelinechannel--bigquery-create-dataset-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-dataset_id + pipelinechannel--bigquery-create-dataset-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-project_id + pipelinechannel--get-fte-suffix-Output: + componentInputParameter: pipelinechannel--get-fte-suffix-Output + pipelinechannel--get-value-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-value + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n CREATE MODEL `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-dataset_id']}}.model_{{$.inputs.parameters['pipelinechannel--get-value-Output']}}`\n\ + \ OPTIONS (\n model_type = 'ARIMA_PLUS',\n \ + \ time_series_timestamp_col = '{{$.inputs.parameters['pipelinechannel--time_column']}}',\n\ + \ time_series_id_col = '{{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}',\n\ + \ time_series_data_col = '{{$.inputs.parameters['pipelinechannel--target_column']}}',\n\ + \ horizon = @forecast_horizon,\n auto_arima\ + \ = True,\n auto_arima_max_order = @max_order,\n \ + \ data_frequency = @data_granularity_unit,\n holiday_region\ + \ = 'GLOBAL',\n clean_spikes_and_dips = True,\n \ + \ adjust_step_changes = True,\n decompose_time_series\ + \ = True\n ) AS\n SELECT\n {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}},\n\ + \ {{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ {{$.inputs.parameters['pipelinechannel--target_column']}},\n\ + \ FROM `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-dataset_id']}}.fte_time_series_output_{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}`\n\ + \ WHERE\n UPPER(split__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}})\ + \ IN UNNEST(@splits)\n AND TIMESTAMP({{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ < @start_time\n " + query_parameters: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-serialized-query-parameters-2 + taskInfo: + name: create-eval-model + query-with-retry-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-query-with-retry-2 + dependentTasks: + - build-job-configuration-query-3 + - build-serialized-query-parameters-2 + - query-with-retry + inputs: + parameters: + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query-3 + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--bigquery-create-dataset-2-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-dataset_id + pipelinechannel--bigquery-create-dataset-2-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-project_id + pipelinechannel--forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + pipelinechannel--get-fte-suffix-Output: + componentInputParameter: pipelinechannel--get-fte-suffix-Output + pipelinechannel--query-with-retry-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: query-with-retry + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n SELECT\n @start_time AS predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ AVG(mean_absolute_error) AS MAE,\n AVG(mean_squared_error)\ + \ AS MSE,\n AVG(mean_absolute_percentage_error) AS MAPE,\n\ + \ @prediction_count AS prediction_count,\n FROM ML.EVALUATE(\n\ + \ MODEL `{{$.inputs.parameters['pipelinechannel--query-with-retry-Output']}}`,\n\ + \ TABLE `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-dataset_id']}}.fte_time_series_output_{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}`,\n\ + \ STRUCT(True AS perform_aggregation, {{$.inputs.parameters['pipelinechannel--forecast_horizon']}}\ + \ as horizon))\n " + query_parameters: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-serialized-query-parameters-2 + taskInfo: + name: append-evaluation-metrics + query-with-retry-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-query-with-retry-3 + dependentTasks: + - build-job-configuration-query-4 + - build-serialized-query-parameters-2 + - query-with-retry + inputs: + parameters: + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query-4 + location: + componentInputParameter: pipelinechannel--get-table-location-Output + pipelinechannel--bigquery-create-dataset-2-dataset_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-dataset_id + pipelinechannel--bigquery-create-dataset-2-project_id: + componentInputParameter: pipelinechannel--bigquery-create-dataset-2-project_id + pipelinechannel--forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + pipelinechannel--get-fte-suffix-Output: + componentInputParameter: pipelinechannel--get-fte-suffix-Output + pipelinechannel--query-with-retry-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: query-with-retry + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n SELECT\n CAST(actual.{{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\ + \ AS STRING)\n AS {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}},\n\ + \ TIMESTAMP(actual.{{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ AS {{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ @start_time AS predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ CAST(actual.{{$.inputs.parameters['pipelinechannel--target_column']}}\ + \ AS FLOAT64) AS {{$.inputs.parameters['pipelinechannel--target_column']}},\n\ + \ STRUCT(pred.forecast_value AS value) AS predicted_{{$.inputs.parameters['pipelinechannel--target_column']}},\n\ + \ FROM\n ML.FORECAST(\n MODEL `{{$.inputs.parameters['pipelinechannel--query-with-retry-Output']}}`,\n\ + \ STRUCT({{$.inputs.parameters['pipelinechannel--forecast_horizon']}}\ + \ AS horizon)) pred\n JOIN `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-2-dataset_id']}}.fte_time_series_output_{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}`\ + \ actual\n ON\n pred.forecast_timestamp = TIMESTAMP(actual.{{$.inputs.parameters['pipelinechannel--time_column']}})\n\ + \ AND pred.{{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\n\ + \ = actual.{{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\n\ + \ " + query_parameters: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-serialized-query-parameters-2 + taskInfo: + name: append-evaluated-examples + inputDefinitions: + parameters: + pipelinechannel--bigquery-create-dataset-2-dataset_id: + parameterType: STRING + pipelinechannel--bigquery-create-dataset-2-project_id: + parameterType: STRING + pipelinechannel--bigquery-create-dataset-dataset_id: + parameterType: STRING + pipelinechannel--bigquery-create-dataset-project_id: + parameterType: STRING + pipelinechannel--bigquery-list-rows-Output: + parameterType: LIST + pipelinechannel--bigquery-list-rows-Output-loop-item: + parameterType: STRUCT + pipelinechannel--data_granularity_unit: + parameterType: STRING + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--get-fte-suffix-Output: + parameterType: STRING + pipelinechannel--get-table-location-Output: + parameterType: STRING + pipelinechannel--max_order: + parameterType: NUMBER_INTEGER + pipelinechannel--project: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--table-to-uri-2-dataset_id: + parameterType: STRING + pipelinechannel--table-to-uri-2-project_id: + parameterType: STRING + pipelinechannel--table-to-uri-2-table_id: + parameterType: STRING + pipelinechannel--table-to-uri-dataset_id: + parameterType: STRING + pipelinechannel--table-to-uri-project_id: + parameterType: STRING + pipelinechannel--table-to-uri-table_id: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_identifier_column: + parameterType: STRING + comp-get-fte-suffix: + executorLabel: exec-get-fte-suffix + inputDefinitions: + parameters: + bigquery_staging_full_dataset_id: + parameterType: STRING + fte_table: + parameterType: STRING + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-table-location: + executorLabel: exec-get-table-location + inputDefinitions: + parameters: + default_location: + defaultValue: '' + description: Location to return if no table was given. + isOptional: true + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + table: + description: The BigQuery table to get a location for. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-value: + executorLabel: exec-get-value + inputDefinitions: + parameters: + d: + parameterType: STRUCT + key: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-window-query-priority: + executorLabel: exec-get-window-query-priority + inputDefinitions: + parameters: + max_interactive: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + window: + parameterType: STRUCT + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-maybe-replace-with-default: + executorLabel: exec-maybe-replace-with-default + inputDefinitions: + parameters: + default: + defaultValue: '' + isOptional: true + parameterType: STRING + value: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-query-with-retry: + executorLabel: exec-query-with-retry + inputDefinitions: + parameters: + destination_uri: + defaultValue: '' + description: Optional BigQuery URI to output if the query succeeds. + isOptional: true + parameterType: STRING + job_configuration_query: + description: Additional query job configurations. + isOptional: true + parameterType: STRUCT + location: + description: The GCP region. + parameterType: STRING + max_retry_count: + defaultValue: 5.0 + description: Maximum number of times to retry the query. + isOptional: true + parameterType: NUMBER_INTEGER + project: + description: The GCP project. + parameterType: STRING + query: + description: The query to run. + parameterType: STRING + query_parameters: + description: A list of query parameters. + isOptional: true + parameterType: LIST + retry_wait_seconds: + defaultValue: 10.0 + description: 'Approximate initial number of seconds to wait before + + making another query attempt with exponential backoff.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-query-with-retry-2: + executorLabel: exec-query-with-retry-2 + inputDefinitions: + parameters: + destination_uri: + defaultValue: '' + description: Optional BigQuery URI to output if the query succeeds. + isOptional: true + parameterType: STRING + job_configuration_query: + description: Additional query job configurations. + isOptional: true + parameterType: STRUCT + location: + description: The GCP region. + parameterType: STRING + max_retry_count: + defaultValue: 5.0 + description: Maximum number of times to retry the query. + isOptional: true + parameterType: NUMBER_INTEGER + project: + description: The GCP project. + parameterType: STRING + query: + description: The query to run. + parameterType: STRING + query_parameters: + description: A list of query parameters. + isOptional: true + parameterType: LIST + retry_wait_seconds: + defaultValue: 10.0 + description: 'Approximate initial number of seconds to wait before + + making another query attempt with exponential backoff.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-query-with-retry-3: + executorLabel: exec-query-with-retry-3 + inputDefinitions: + parameters: + destination_uri: + defaultValue: '' + description: Optional BigQuery URI to output if the query succeeds. + isOptional: true + parameterType: STRING + job_configuration_query: + description: Additional query job configurations. + isOptional: true + parameterType: STRUCT + location: + description: The GCP region. + parameterType: STRING + max_retry_count: + defaultValue: 5.0 + description: Maximum number of times to retry the query. + isOptional: true + parameterType: NUMBER_INTEGER + project: + description: The GCP project. + parameterType: STRING + query: + description: The query to run. + parameterType: STRING + query_parameters: + description: A list of query parameters. + isOptional: true + parameterType: LIST + retry_wait_seconds: + defaultValue: 10.0 + description: 'Approximate initial number of seconds to wait before + + making another query attempt with exponential backoff.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-table-to-uri: + executorLabel: exec-table-to-uri + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-table-to-uri-2: + executorLabel: exec-table-to-uri-2 + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-validate-inputs: + executorLabel: exec-validate-inputs + inputDefinitions: + parameters: + bigquery_destination_uri: + isOptional: true + parameterType: STRING + data_granularity_unit: + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + isOptional: true + parameterType: STRING + data_source_csv_filenames: + isOptional: true + parameterType: STRING + optimization_objective: + isOptional: true + parameterType: STRING + predefined_split_key: + isOptional: true + parameterType: STRING + source_model_uri: + isOptional: true + parameterType: STRING + target_column: + isOptional: true + parameterType: STRING + test_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + isOptional: true + parameterType: STRING + time_series_identifier_column: + isOptional: true + parameterType: STRING + timestamp_split_key: + isOptional: true + parameterType: STRING + training_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + window_column: + isOptional: true + parameterType: STRING + window_max_count: + isOptional: true + parameterType: NUMBER_INTEGER + window_stride_length: + isOptional: true + parameterType: NUMBER_INTEGER + comp-wrapped-in-list: + executorLabel: exec-wrapped-in-list + inputDefinitions: + parameters: + value: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: LIST +deploymentSpec: + executors: + exec-bigquery-create-dataset: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_create_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_create_dataset(\n project: str,\n location: str,\n\ + \ dataset: str,\n exists_ok: bool = False,\n) -> NamedTuple('Outputs',\ + \ [('project_id', str), ('dataset_id', str)]):\n \"\"\"Creates a BigQuery\ + \ dataset.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n\n from google.cloud import bigquery\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n ref\ + \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ + \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ + \ ref.project, ref.dataset_id)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-create-dataset-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_create_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_create_dataset(\n project: str,\n location: str,\n\ + \ dataset: str,\n exists_ok: bool = False,\n) -> NamedTuple('Outputs',\ + \ [('project_id', str), ('dataset_id', str)]):\n \"\"\"Creates a BigQuery\ + \ dataset.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n\n from google.cloud import bigquery\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n ref\ + \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ + \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ + \ ref.project, ref.dataset_id)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-create-model-job: + container: + args: + - --type + - BigqueryCreateModelJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.create_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-bigquery-delete-dataset-with-prefix: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_delete_dataset_with_prefix + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_delete_dataset_with_prefix(\n project: str,\n \ + \ dataset_prefix: str,\n delete_contents: bool = False,\n) -> None:\n\ + \ \"\"\"Deletes all BigQuery datasets matching the given prefix.\"\"\"\n\ + \ # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project)\n for dataset in client.list_datasets(project=project):\n\ + \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ + \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-list-rows: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_list_rows + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_list_rows(\n project: str,\n location: str,\n\ + \ table: dsl.Input[dsl.Artifact],\n) -> List[Dict[str, str]]:\n \"\"\ + \"Lists the rows of the given BigQuery table.\n\n Args:\n project: The\ + \ GCP project.\n location: The GCP region.\n table: A google.BQTable\ + \ artifact.\n\n Returns:\n A list of dicts representing BigQuery rows.\ + \ Rows are keyed by column, and\n all values are stored as strings.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n metadata\ + \ = table.metadata\n rows = client.list_rows('.'.join(\n [metadata['projectId'],\ + \ metadata['datasetId'], metadata['tableId']]))\n result = []\n for row\ + \ in rows:\n result.append({col: str(value) for col, value in dict(row).items()})\n\ + \ return result\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-list-rows-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_list_rows + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_list_rows(\n project: str,\n location: str,\n\ + \ table: dsl.Input[dsl.Artifact],\n) -> List[Dict[str, str]]:\n \"\"\ + \"Lists the rows of the given BigQuery table.\n\n Args:\n project: The\ + \ GCP project.\n location: The GCP region.\n table: A google.BQTable\ + \ artifact.\n\n Returns:\n A list of dicts representing BigQuery rows.\ + \ Rows are keyed by column, and\n all values are stored as strings.\n\ + \ \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n metadata\ + \ = table.metadata\n rows = client.list_rows('.'.join(\n [metadata['projectId'],\ + \ metadata['datasetId'], metadata['tableId']]))\n result = []\n for row\ + \ in rows:\n result.append({col: str(value) for col, value in dict(row).items()})\n\ + \ return result\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-query-job: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-bigquery-query-job-2: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-bigquery-query-job-3: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-bigquery-query-job-4: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-bigquery-query-job-5: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-build-job-configuration-query: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-job-configuration-query-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-job-configuration-query-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-job-configuration-query-4: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-job-configuration-query-5: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-job-configuration-query-6: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-serialized-query-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_serialized_query_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_serialized_query_parameters(\n forecast_horizon: Optional[int]\ + \ = None,\n forecast_horizon_off_by_one: bool = False,\n data_granularity_unit:\ + \ Optional[str] = None,\n splits: Optional[List[str]] = None,\n window:\ + \ Optional[Dict[str, str]] = None,\n max_order: Optional[int] = None,\n\ + ) -> list: # pylint: disable=g-bare-generic\n \"\"\"Creates configuration\ + \ JSON objects for BQML queries.\n\n All query parameters will be stored\ + \ in a list of QueryParameter objects:\n https://cloud.google.com/bigquery/docs/reference/rest/v2/QueryParameter\n\ + \n Args:\n forecast_horizon: The number of time periods into the future\ + \ for which\n forecasts will be created. Future periods start after\ + \ the latest timestamp\n for each time series.\n forecast_horizon_off_by_one:\ + \ If True, subtract 1 from the forecast horizon\n in the query parameters.\n\ + \ data_granularity_unit: The data granularity unit. Accepted values are:\n\ + \ minute, hour, day, week, month, year.\n splits: Dataset splits\ + \ to be used to train the model.\n window: Dict containing information\ + \ about the forecast window the model\n should have. If no window is\ + \ provided, the window will start after the\n latest period in the\ + \ available data.\n max_order: Integer between 1 and 5 representing the\ + \ size of the parameter\n search space for ARIMA_PLUS. 5 would result\ + \ in the highest accuracy model,\n but also the longest training runtime.\n\ + \n Returns:\n A list of QueryParameters.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import datetime\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n # Maps Vertex Forecasting time units to BQML time units.\n unit_map\ + \ = {\n 'minute': 'per_minute',\n 'hour': 'hourly',\n 'day':\ + \ 'daily',\n 'week': 'weekly',\n 'month': 'monthly',\n 'year':\ + \ 'yearly',\n }\n query_parameters = []\n if data_granularity_unit is\ + \ not None:\n if data_granularity_unit.lower() not in unit_map:\n \ + \ raise ValueError(\n f'{data_granularity_unit} is not a valid\ + \ time unit. '\n f'Must be one of: {\", \".join(unit_map.keys())}')\n\ + \ query_parameters.append({\n 'name': 'data_granularity_unit',\n\ + \ 'parameterType': {\n 'type': 'STRING'\n },\n\ + \ 'parameterValue': {\n 'value': unit_map[data_granularity_unit.lower()],\n\ + \ },\n })\n if max_order is not None:\n query_parameters.append({\n\ + \ 'name': 'max_order',\n 'parameterType': {\n 'type':\ + \ 'INTEGER'\n },\n 'parameterValue': {\n 'value':\ + \ str(max_order)\n },\n })\n if forecast_horizon is not None:\n\ + \ if forecast_horizon_off_by_one:\n forecast_horizon -= 1\n query_parameters.append({\n\ + \ 'name': 'forecast_horizon',\n 'parameterType': {\n \ + \ 'type': 'INTEGER'\n },\n 'parameterValue': {\n \ + \ 'value': str(forecast_horizon)\n },\n })\n if splits\ + \ is not None:\n query_parameters.append({\n 'name': 'splits',\n\ + \ 'parameterType': {\n 'type': 'ARRAY',\n 'arrayType':\ + \ {\n 'type': 'STRING'\n },\n },\n \ + \ 'parameterValue': {\n 'arrayValues': [{\n \ + \ 'value': split\n } for split in splits],\n },\n \ + \ })\n\n if window is not None:\n query_parameters.append({\n \ + \ 'name': 'prediction_count',\n 'parameterType': {\n \ + \ 'type': 'INTEGER'\n },\n 'parameterValue': {\n \ + \ 'value': window['count']\n },\n })\n\n start_time = window['start_time']\ + \ if window else str(datetime.datetime.max)\n query_parameters.append({\n\ + \ 'name': 'start_time',\n 'parameterType': {\n 'type':\ + \ 'TIMESTAMP'\n },\n 'parameterValue': {\n 'value': start_time\n\ + \ },\n })\n return query_parameters\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-serialized-query-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_serialized_query_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_serialized_query_parameters(\n forecast_horizon: Optional[int]\ + \ = None,\n forecast_horizon_off_by_one: bool = False,\n data_granularity_unit:\ + \ Optional[str] = None,\n splits: Optional[List[str]] = None,\n window:\ + \ Optional[Dict[str, str]] = None,\n max_order: Optional[int] = None,\n\ + ) -> list: # pylint: disable=g-bare-generic\n \"\"\"Creates configuration\ + \ JSON objects for BQML queries.\n\n All query parameters will be stored\ + \ in a list of QueryParameter objects:\n https://cloud.google.com/bigquery/docs/reference/rest/v2/QueryParameter\n\ + \n Args:\n forecast_horizon: The number of time periods into the future\ + \ for which\n forecasts will be created. Future periods start after\ + \ the latest timestamp\n for each time series.\n forecast_horizon_off_by_one:\ + \ If True, subtract 1 from the forecast horizon\n in the query parameters.\n\ + \ data_granularity_unit: The data granularity unit. Accepted values are:\n\ + \ minute, hour, day, week, month, year.\n splits: Dataset splits\ + \ to be used to train the model.\n window: Dict containing information\ + \ about the forecast window the model\n should have. If no window is\ + \ provided, the window will start after the\n latest period in the\ + \ available data.\n max_order: Integer between 1 and 5 representing the\ + \ size of the parameter\n search space for ARIMA_PLUS. 5 would result\ + \ in the highest accuracy model,\n but also the longest training runtime.\n\ + \n Returns:\n A list of QueryParameters.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import datetime\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n # Maps Vertex Forecasting time units to BQML time units.\n unit_map\ + \ = {\n 'minute': 'per_minute',\n 'hour': 'hourly',\n 'day':\ + \ 'daily',\n 'week': 'weekly',\n 'month': 'monthly',\n 'year':\ + \ 'yearly',\n }\n query_parameters = []\n if data_granularity_unit is\ + \ not None:\n if data_granularity_unit.lower() not in unit_map:\n \ + \ raise ValueError(\n f'{data_granularity_unit} is not a valid\ + \ time unit. '\n f'Must be one of: {\", \".join(unit_map.keys())}')\n\ + \ query_parameters.append({\n 'name': 'data_granularity_unit',\n\ + \ 'parameterType': {\n 'type': 'STRING'\n },\n\ + \ 'parameterValue': {\n 'value': unit_map[data_granularity_unit.lower()],\n\ + \ },\n })\n if max_order is not None:\n query_parameters.append({\n\ + \ 'name': 'max_order',\n 'parameterType': {\n 'type':\ + \ 'INTEGER'\n },\n 'parameterValue': {\n 'value':\ + \ str(max_order)\n },\n })\n if forecast_horizon is not None:\n\ + \ if forecast_horizon_off_by_one:\n forecast_horizon -= 1\n query_parameters.append({\n\ + \ 'name': 'forecast_horizon',\n 'parameterType': {\n \ + \ 'type': 'INTEGER'\n },\n 'parameterValue': {\n \ + \ 'value': str(forecast_horizon)\n },\n })\n if splits\ + \ is not None:\n query_parameters.append({\n 'name': 'splits',\n\ + \ 'parameterType': {\n 'type': 'ARRAY',\n 'arrayType':\ + \ {\n 'type': 'STRING'\n },\n },\n \ + \ 'parameterValue': {\n 'arrayValues': [{\n \ + \ 'value': split\n } for split in splits],\n },\n \ + \ })\n\n if window is not None:\n query_parameters.append({\n \ + \ 'name': 'prediction_count',\n 'parameterType': {\n \ + \ 'type': 'INTEGER'\n },\n 'parameterValue': {\n \ + \ 'value': window['count']\n },\n })\n\n start_time = window['start_time']\ + \ if window else str(datetime.datetime.max)\n query_parameters.append({\n\ + \ 'name': 'start_time',\n 'parameterType': {\n 'type':\ + \ 'TIMESTAMP'\n },\n 'parameterValue': {\n 'value': start_time\n\ + \ },\n })\n return query_parameters\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-serialized-query-parameters-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_serialized_query_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_serialized_query_parameters(\n forecast_horizon: Optional[int]\ + \ = None,\n forecast_horizon_off_by_one: bool = False,\n data_granularity_unit:\ + \ Optional[str] = None,\n splits: Optional[List[str]] = None,\n window:\ + \ Optional[Dict[str, str]] = None,\n max_order: Optional[int] = None,\n\ + ) -> list: # pylint: disable=g-bare-generic\n \"\"\"Creates configuration\ + \ JSON objects for BQML queries.\n\n All query parameters will be stored\ + \ in a list of QueryParameter objects:\n https://cloud.google.com/bigquery/docs/reference/rest/v2/QueryParameter\n\ + \n Args:\n forecast_horizon: The number of time periods into the future\ + \ for which\n forecasts will be created. Future periods start after\ + \ the latest timestamp\n for each time series.\n forecast_horizon_off_by_one:\ + \ If True, subtract 1 from the forecast horizon\n in the query parameters.\n\ + \ data_granularity_unit: The data granularity unit. Accepted values are:\n\ + \ minute, hour, day, week, month, year.\n splits: Dataset splits\ + \ to be used to train the model.\n window: Dict containing information\ + \ about the forecast window the model\n should have. If no window is\ + \ provided, the window will start after the\n latest period in the\ + \ available data.\n max_order: Integer between 1 and 5 representing the\ + \ size of the parameter\n search space for ARIMA_PLUS. 5 would result\ + \ in the highest accuracy model,\n but also the longest training runtime.\n\ + \n Returns:\n A list of QueryParameters.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import datetime\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n # Maps Vertex Forecasting time units to BQML time units.\n unit_map\ + \ = {\n 'minute': 'per_minute',\n 'hour': 'hourly',\n 'day':\ + \ 'daily',\n 'week': 'weekly',\n 'month': 'monthly',\n 'year':\ + \ 'yearly',\n }\n query_parameters = []\n if data_granularity_unit is\ + \ not None:\n if data_granularity_unit.lower() not in unit_map:\n \ + \ raise ValueError(\n f'{data_granularity_unit} is not a valid\ + \ time unit. '\n f'Must be one of: {\", \".join(unit_map.keys())}')\n\ + \ query_parameters.append({\n 'name': 'data_granularity_unit',\n\ + \ 'parameterType': {\n 'type': 'STRING'\n },\n\ + \ 'parameterValue': {\n 'value': unit_map[data_granularity_unit.lower()],\n\ + \ },\n })\n if max_order is not None:\n query_parameters.append({\n\ + \ 'name': 'max_order',\n 'parameterType': {\n 'type':\ + \ 'INTEGER'\n },\n 'parameterValue': {\n 'value':\ + \ str(max_order)\n },\n })\n if forecast_horizon is not None:\n\ + \ if forecast_horizon_off_by_one:\n forecast_horizon -= 1\n query_parameters.append({\n\ + \ 'name': 'forecast_horizon',\n 'parameterType': {\n \ + \ 'type': 'INTEGER'\n },\n 'parameterValue': {\n \ + \ 'value': str(forecast_horizon)\n },\n })\n if splits\ + \ is not None:\n query_parameters.append({\n 'name': 'splits',\n\ + \ 'parameterType': {\n 'type': 'ARRAY',\n 'arrayType':\ + \ {\n 'type': 'STRING'\n },\n },\n \ + \ 'parameterValue': {\n 'arrayValues': [{\n \ + \ 'value': split\n } for split in splits],\n },\n \ + \ })\n\n if window is not None:\n query_parameters.append({\n \ + \ 'name': 'prediction_count',\n 'parameterType': {\n \ + \ 'type': 'INTEGER'\n },\n 'parameterValue': {\n \ + \ 'value': window['count']\n },\n })\n\n start_time = window['start_time']\ + \ if window else str(datetime.datetime.max)\n query_parameters.append({\n\ + \ 'name': 'start_time',\n 'parameterType': {\n 'type':\ + \ 'TIMESTAMP'\n },\n 'parameterValue': {\n 'value': start_time\n\ + \ },\n })\n return query_parameters\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-cond: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - cond + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef cond(predicate: bool, true_str: str, false_str: str) -> str:\n\ + \ \"\"\"Returns true_str if predicate is true, else false_str.\"\"\"\n\ + \ return true_str if predicate else false_str\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-create-metrics-artifact: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - create_metrics_artifact + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef create_metrics_artifact(\n metrics_rows: List[Dict[str, str]],\n\ + \ evaluation_metrics: dsl.Output[dsl.Metrics],\n) -> None:\n \"\"\"\ + Converts the rows of a metrics table into an Artifact.\"\"\"\n metric_name_map\ + \ = {\n 'MAE': 'meanAbsoluteError',\n 'RMSE': 'rootMeanSquaredError',\n\ + \ 'MAPE': 'meanAbsolutePercentageError',\n }\n metrics = {metric_name_map[k]:\ + \ v for k, v in dict(metrics_rows[0]).items()}\n evaluation_metrics.metadata\ + \ = metrics\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + exec-get-fte-suffix: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_fte_suffix + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_fte_suffix(\n project: str,\n location: str,\n bigquery_staging_full_dataset_id:\ + \ str,\n fte_table: str,\n) -> str:\n \"\"\"Infers the FTE suffix from\ + \ the intermediate FTE table name.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n for\ + \ table in client.list_tables(bigquery_staging_full_dataset_id):\n if\ + \ table.table_id.startswith(fte_table):\n return table.table_id[len(fte_table)\ + \ + 1:]\n raise ValueError(\n f'No FTE output tables found in {bigquery_staging_full_dataset_id}.')\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-table-location: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_table_location + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_table_location(\n project: str,\n table: Optional[str],\n\ + \ default_location: str = '',\n) -> str:\n \"\"\"Returns the region\ + \ the given table belongs to.\n\n Args:\n project: The GCP project.\n\ + \ table: The BigQuery table to get a location for.\n default_location:\ + \ Location to return if no table was given.\n\n Returns:\n A GCP region\ + \ or multi-region.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n if not table:\n return default_location\n\n client = bigquery.Client(project=project)\n\ + \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ + \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ + \ return client.get_table(table).location\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-value: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_value + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_value(d: Dict[str, str], key: str) -> str:\n return d[key]\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-window-query-priority: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_window_query_priority + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_window_query_priority(\n window: Dict[str, str],\n \ + \ max_interactive: int = 100,\n) -> str:\n \"\"\"Returns a query priority\ + \ depending on the window number.\"\"\"\n if int(window['window_number'])\ + \ <= max_interactive:\n return 'INTERACTIVE'\n else:\n return 'BATCH'\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-maybe-replace-with-default: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - maybe_replace_with_default + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef maybe_replace_with_default(value: str, default: str = '') ->\ + \ str:\n \"\"\"Replaces string with another value if it is a dash.\"\"\"\ + \n return default if not value else value\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-query-with-retry: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - query_with_retry + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef query_with_retry(\n project: str,\n location: str,\n \ + \ query: str,\n query_parameters: Optional[list] = None, # pylint:\ + \ disable=g-bare-generic\n job_configuration_query: Optional[dict] =\ + \ None, # pylint: disable=g-bare-generic\n max_retry_count: int = 5,\n\ + \ retry_wait_seconds: int = 10, # Waits up to 4 minutes before 5th retry.\n\ + \ destination_uri: str = '',\n) -> str:\n \"\"\"Runs a query and retries\ + \ on failure.\n\n Args:\n project: The GCP project.\n location: The\ + \ GCP region.\n query: The query to run.\n query_parameters: A list\ + \ of query parameters.\n job_configuration_query: Additional query job\ + \ configurations.\n max_retry_count: Maximum number of times to retry\ + \ the query.\n retry_wait_seconds: Approximate initial number of seconds\ + \ to wait before\n making another query attempt with exponential backoff.\n\ + \ destination_uri: Optional BigQuery URI to output if the query succeeds.\n\ + \n Returns:\n The given destination URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import logging\n import random\n import time\n\n from google.api_core\ + \ import exceptions\n from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n query_parameters = query_parameters or []\n job_configuration_query\ + \ = job_configuration_query or {}\n client = bigquery.Client(project=project,\ + \ location=location)\n\n job_configuration_query['queryParameters'] = query_parameters\n\ + \ job_config = bigquery.QueryJobConfig.from_api_repr(\n {'query':\ + \ job_configuration_query})\n retry_count = 0\n while True:\n try:\n\ + \ client.query(query, job_config=job_config).result()\n break\n\ + \ except (exceptions.BadRequest, exceptions.Forbidden) as e:\n if\ + \ retry_count >= max_retry_count:\n logging.info('Maximum retries\ + \ reached.')\n raise\n wait_time = (\n retry_wait_seconds\ + \ * (2 ** retry_count) * random.uniform(1, 1.5))\n logging.info(\n\ + \ 'Query failed with %s. Retrying after %d seconds.', e, wait_time)\n\ + \ time.sleep(wait_time)\n retry_count += 1\n return destination_uri\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-query-with-retry-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - query_with_retry + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef query_with_retry(\n project: str,\n location: str,\n \ + \ query: str,\n query_parameters: Optional[list] = None, # pylint:\ + \ disable=g-bare-generic\n job_configuration_query: Optional[dict] =\ + \ None, # pylint: disable=g-bare-generic\n max_retry_count: int = 5,\n\ + \ retry_wait_seconds: int = 10, # Waits up to 4 minutes before 5th retry.\n\ + \ destination_uri: str = '',\n) -> str:\n \"\"\"Runs a query and retries\ + \ on failure.\n\n Args:\n project: The GCP project.\n location: The\ + \ GCP region.\n query: The query to run.\n query_parameters: A list\ + \ of query parameters.\n job_configuration_query: Additional query job\ + \ configurations.\n max_retry_count: Maximum number of times to retry\ + \ the query.\n retry_wait_seconds: Approximate initial number of seconds\ + \ to wait before\n making another query attempt with exponential backoff.\n\ + \ destination_uri: Optional BigQuery URI to output if the query succeeds.\n\ + \n Returns:\n The given destination URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import logging\n import random\n import time\n\n from google.api_core\ + \ import exceptions\n from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n query_parameters = query_parameters or []\n job_configuration_query\ + \ = job_configuration_query or {}\n client = bigquery.Client(project=project,\ + \ location=location)\n\n job_configuration_query['queryParameters'] = query_parameters\n\ + \ job_config = bigquery.QueryJobConfig.from_api_repr(\n {'query':\ + \ job_configuration_query})\n retry_count = 0\n while True:\n try:\n\ + \ client.query(query, job_config=job_config).result()\n break\n\ + \ except (exceptions.BadRequest, exceptions.Forbidden) as e:\n if\ + \ retry_count >= max_retry_count:\n logging.info('Maximum retries\ + \ reached.')\n raise\n wait_time = (\n retry_wait_seconds\ + \ * (2 ** retry_count) * random.uniform(1, 1.5))\n logging.info(\n\ + \ 'Query failed with %s. Retrying after %d seconds.', e, wait_time)\n\ + \ time.sleep(wait_time)\n retry_count += 1\n return destination_uri\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-query-with-retry-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - query_with_retry + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef query_with_retry(\n project: str,\n location: str,\n \ + \ query: str,\n query_parameters: Optional[list] = None, # pylint:\ + \ disable=g-bare-generic\n job_configuration_query: Optional[dict] =\ + \ None, # pylint: disable=g-bare-generic\n max_retry_count: int = 5,\n\ + \ retry_wait_seconds: int = 10, # Waits up to 4 minutes before 5th retry.\n\ + \ destination_uri: str = '',\n) -> str:\n \"\"\"Runs a query and retries\ + \ on failure.\n\n Args:\n project: The GCP project.\n location: The\ + \ GCP region.\n query: The query to run.\n query_parameters: A list\ + \ of query parameters.\n job_configuration_query: Additional query job\ + \ configurations.\n max_retry_count: Maximum number of times to retry\ + \ the query.\n retry_wait_seconds: Approximate initial number of seconds\ + \ to wait before\n making another query attempt with exponential backoff.\n\ + \ destination_uri: Optional BigQuery URI to output if the query succeeds.\n\ + \n Returns:\n The given destination URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import logging\n import random\n import time\n\n from google.api_core\ + \ import exceptions\n from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n query_parameters = query_parameters or []\n job_configuration_query\ + \ = job_configuration_query or {}\n client = bigquery.Client(project=project,\ + \ location=location)\n\n job_configuration_query['queryParameters'] = query_parameters\n\ + \ job_config = bigquery.QueryJobConfig.from_api_repr(\n {'query':\ + \ job_configuration_query})\n retry_count = 0\n while True:\n try:\n\ + \ client.query(query, job_config=job_config).result()\n break\n\ + \ except (exceptions.BadRequest, exceptions.Forbidden) as e:\n if\ + \ retry_count >= max_retry_count:\n logging.info('Maximum retries\ + \ reached.')\n raise\n wait_time = (\n retry_wait_seconds\ + \ * (2 ** retry_count) * random.uniform(1, 1.5))\n logging.info(\n\ + \ 'Query failed with %s. Retrying after %d seconds.', e, wait_time)\n\ + \ time.sleep(wait_time)\n retry_count += 1\n return destination_uri\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-table-to-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-table-to-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-validate-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - validate_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef validate_inputs(\n time_column: Optional[str] = None,\n \ + \ time_series_identifier_column: Optional[str] = None,\n target_column:\ + \ Optional[str] = None,\n data_source_bigquery_table_path: Optional[str]\ + \ = None,\n training_fraction: Optional[float] = None,\n validation_fraction:\ + \ Optional[float] = None,\n test_fraction: Optional[float] = None,\n\ + \ predefined_split_key: Optional[str] = None,\n timestamp_split_key:\ + \ Optional[str] = None,\n data_source_csv_filenames: Optional[str] =\ + \ None,\n source_model_uri: Optional[str] = None,\n bigquery_destination_uri:\ + \ Optional[str] = None,\n window_column: Optional[str] = None,\n window_stride_length:\ + \ Optional[int] = None,\n window_max_count: Optional[int] = None,\n \ + \ optimization_objective: Optional[str] = None,\n data_granularity_unit:\ + \ Optional[str] = None,\n) -> None:\n \"\"\"Checks training pipeline input\ + \ parameters are valid.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import re\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n project_pattern = r'([a-z0-9.-]+:)?[a-z][a-z0-9-_]{4,28}[a-z0-9]'\n\ + \ dataset_pattern = r'[a-zA-Z0-9_]+'\n table_pattern = r'[^\\.\\:`]+'\n\ + \ dataset_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}')\n\ + \ table_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}[.:]{table_pattern}')\n\ + \n # Validate BigQuery column and dataset names.\n bigquery_column_parameters\ + \ = [\n time_column,\n time_series_identifier_column,\n target_column,\n\ + \ ]\n column_pattern = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]{1,300}')\n \ + \ for column in bigquery_column_parameters:\n if column and not column_pattern.fullmatch(column):\n\ + \ raise ValueError(f'Invalid column name: {column}.')\n if (bigquery_destination_uri\ + \ and\n not dataset_uri_pattern.fullmatch(bigquery_destination_uri)):\n\ + \ raise ValueError(\n f'Invalid BigQuery dataset URI: {bigquery_destination_uri}.')\n\ + \ if (source_model_uri and not table_uri_pattern.fullmatch(source_model_uri)):\n\ + \ raise ValueError(f'Invalid BigQuery table URI: {source_model_uri}.')\n\ + \n # Validate data source.\n data_source_count = sum([bool(source) for\ + \ source in [\n data_source_bigquery_table_path, data_source_csv_filenames]])\n\ + \ if data_source_count > 1:\n raise ValueError(f'Expected 1 data source,\ + \ found {data_source_count}.')\n if (data_source_bigquery_table_path\n\ + \ and not table_uri_pattern.fullmatch(data_source_bigquery_table_path)):\n\ + \ raise ValueError(\n f'Invalid BigQuery table URI: {data_source_bigquery_table_path}.')\n\ + \ gcs_path_pattern = re.compile(r'gs:\\/\\/(.+)\\/([^\\/]+)')\n if data_source_csv_filenames:\n\ + \ csv_list = [filename.strip()\n for filename in data_source_csv_filenames.split(',')]\n\ + \ for gcs_path in csv_list:\n if not gcs_path_pattern.fullmatch(gcs_path):\n\ + \ raise ValueError(f'Invalid path to CSV stored in GCS: {gcs_path}.')\n\ + \n # Validate split spec.\n fraction_splits = [\n training_fraction,\n\ + \ validation_fraction,\n test_fraction,\n ]\n fraction_splits\ + \ = [None if fraction == -1 else fraction\n for fraction\ + \ in fraction_splits]\n split_count = sum([\n bool(source)\n \ + \ for source in [predefined_split_key,\n any(fraction_splits)]\n\ + \ ])\n if split_count > 1:\n raise ValueError(f'Expected 1 split type,\ + \ found {split_count}.')\n if (predefined_split_key and\n not column_pattern.fullmatch(predefined_split_key)):\n\ + \ raise ValueError(f'Invalid column name: {predefined_split_key}.')\n\ + \ if any(fraction_splits):\n if not all(fraction_splits):\n raise\ + \ ValueError(\n f'All fractions must be non-zero. Got: {fraction_splits}.')\n\ + \ if sum(fraction_splits) != 1:\n raise ValueError(\n f'Fraction\ + \ splits must sum to 1. Got: {sum(fraction_splits)}.')\n if (timestamp_split_key\ + \ and\n not column_pattern.fullmatch(timestamp_split_key)):\n raise\ + \ ValueError(f'Invalid column name: {timestamp_split_key}.')\n if timestamp_split_key\ + \ and not all(fraction_splits):\n raise ValueError('All fractions must\ + \ be non-zero for timestamp split.')\n\n # Validate window config.\n if\ + \ window_stride_length == -1:\n window_stride_length = None\n if window_max_count\ + \ == -1:\n window_max_count = None\n window_configs = [window_column,\ + \ window_stride_length, window_max_count]\n window_config_count = sum([bool(config)\ + \ for config in window_configs])\n if window_config_count > 1:\n raise\ + \ ValueError(f'Expected 1 window config, found {window_config_count}.')\n\ + \ if window_column and not column_pattern.fullmatch(window_column):\n \ + \ raise ValueError(f'Invalid column name: {window_column}.')\n if window_stride_length\ + \ and (window_stride_length < 1 or\n window_stride_length\ + \ > 1000):\n raise ValueError('Stride must be between 1 and 1000. Got:\ + \ '\n f'{window_stride_length}.')\n if window_max_count\ + \ and (window_max_count < 1000 or\n window_max_count\ + \ > int(1e8)):\n raise ValueError('Max count must be between 1000 and\ + \ 100000000. Got: '\n f'{window_max_count}.')\n\n #\ + \ Validate eval metric.\n valid_optimization_objectives = ['rmse', 'mae',\ + \ 'rmsle']\n if optimization_objective:\n if optimization_objective\ + \ not in valid_optimization_objectives:\n raise ValueError(\n \ + \ 'Optimization objective should be one of the following: '\n \ + \ f'{valid_optimization_objectives}, got: {optimization_objective}.')\n\ + \n # Validate data granularity unit.\n valid_data_granularity_units =\ + \ [\n 'minute', 'hour', 'day', 'week', 'month', 'year']\n if data_granularity_unit:\n\ + \ if data_granularity_unit not in valid_data_granularity_units:\n \ + \ raise ValueError(\n 'Granularity unit should be one of the\ + \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-wrapped-in-list: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - wrapped_in_list + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef wrapped_in_list(value: str) -> List[str]:\n \"\"\"Wraps a string\ + \ in a list.\"\"\"\n return [value]\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 +pipelineInfo: + description: Trains a BQML ARIMA_PLUS model. + name: automl-tabular-bqml-arima-train +root: + dag: + outputs: + artifacts: + create-metrics-artifact-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: create-metrics-artifact-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + bigquery-delete-dataset-with-prefix: + cachingOptions: {} + componentRef: + name: comp-bigquery-delete-dataset-with-prefix + dependentTasks: + - exit-handler-1 + inputs: + parameters: + dataset_prefix: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + delete_contents: + runtimeValue: + constant: true + project: + componentInputParameter: project + taskInfo: + name: delete-tmp-dataset + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + inputs: + parameters: + pipelinechannel--bigquery_destination_uri: + componentInputParameter: bigquery_destination_uri + pipelinechannel--data_granularity_unit: + componentInputParameter: data_granularity_unit + pipelinechannel--data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + pipelinechannel--data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--forecast_horizon: + componentInputParameter: forecast_horizon + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--max_order: + componentInputParameter: max_order + pipelinechannel--override_destination: + componentInputParameter: override_destination + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--time_column: + componentInputParameter: time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: time_series_identifier_column + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--window_column: + componentInputParameter: window_column + pipelinechannel--window_max_count: + componentInputParameter: window_max_count + pipelinechannel--window_stride_length: + componentInputParameter: window_stride_length + taskInfo: + name: exit-handler-1 + inputDefinitions: + parameters: + bigquery_destination_uri: + defaultValue: '' + description: 'URI of the desired destination dataset. If not + + specified, resources will be created under a new dataset in the project. + + Unlike in Vertex Forecasting, all resources will be given hardcoded names + + under this dataset, and the model artifact will also be exported here.' + isOptional: true + parameterType: STRING + data_granularity_unit: + description: 'The data granularity unit. Accepted values are: + + minute, hour, day, week, month, year.' + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + forecast_horizon: + description: 'The number of time periods into the future for which + + forecasts will be created. Future periods start after the latest timestamp + + for each time series.' + parameterType: NUMBER_INTEGER + location: + description: The GCP region for Vertex AI. + parameterType: STRING + max_order: + defaultValue: 5.0 + description: 'Integer between 1 and 5 representing the size of the parameter + + search space for ARIMA_PLUS. 5 would result in the highest accuracy model, + + but also the longest training runtime.' + isOptional: true + parameterType: NUMBER_INTEGER + override_destination: + defaultValue: false + description: 'Whether to overwrite the metrics and evaluated + + examples tables if they already exist. If this is False and the tables + + exist, this pipeline will fail.' + isOptional: true + parameterType: BOOLEAN + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_evaluation: + defaultValue: true + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + target_column: + description: Name of the column that the model is to predict values for. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: float = The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + description: 'Name of the column that identifies time order in the time + + series.' + parameterType: STRING + time_series_identifier_column: + description: 'Name of the column that identifies the time + + series.' + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + window_column: + defaultValue: '' + description: 'Name of the column that should be used to filter input rows. + + The column should contain either booleans or string booleans; if the value + + of the row is True, generate a sliding window from that row.' + isOptional: true + parameterType: STRING + window_max_count: + defaultValue: -1.0 + description: 'Number of rows that should be used to generate input + + examples. If the total row count is larger than this number, the input + + data will be randomly sampled to hit the count.' + isOptional: true + parameterType: NUMBER_INTEGER + window_stride_length: + defaultValue: -1.0 + description: 'Step length used to generate input examples. Every + + window_stride_length rows will be used to generate a sliding window.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + create-metrics-artifact-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml new file mode 100644 index 0000000000..83e39834c4 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_predict_pipeline.yaml @@ -0,0 +1,2169 @@ +# PIPELINE DEFINITION +# Name: prophet-predict +# Description: Creates a batch prediction using a Prophet model. +# Inputs: +# bigquery_destination_uri: str [Default: ''] +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# encryption_spec_key_name: str [Default: ''] +# location: str +# machine_type: str [Default: 'n1-standard-2'] +# max_num_workers: int [Default: 10.0] +# model_name: str +# project: str +# target_column: str +# time_column: str +# time_series_identifier_column: str +components: + comp-bigquery-create-dataset: + executorLabel: exec-bigquery-create-dataset + inputDefinitions: + parameters: + dataset: + parameterType: STRING + exists_ok: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + comp-bigquery-delete-dataset-with-prefix: + executorLabel: exec-bigquery-delete-dataset-with-prefix + inputDefinitions: + parameters: + dataset_prefix: + parameterType: STRING + delete_contents: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + project: + parameterType: STRING + comp-bigquery-query-job: + executorLabel: exec-bigquery-query-job + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bigquery-query-job-2: + executorLabel: exec-bigquery-query-job-2 + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-build-job-configuration-query: + executorLabel: exec-build-job-configuration-query + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-build-job-configuration-query-2: + executorLabel: exec-build-job-configuration-query-2 + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-exit-handler-1: + dag: + tasks: + bigquery-create-dataset: + cachingOptions: {} + componentRef: + name: comp-bigquery-create-dataset + dependentTasks: + - get-table-location + - validate-inputs + inputs: + parameters: + dataset: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: create-tmp-dataset + bigquery-query-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job + dependentTasks: + - build-job-configuration-query + - get-first-valid + - get-table-location + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + pipelinechannel--get-first-valid-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-first-valid + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n WITH\n base_data AS (\n SELECT\ + \ * FROM `{{$.inputs.parameters['pipelinechannel--get-first-valid-Output']}}`\n\ + \ )\n SELECT\n CAST({{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\ + \ AS STRING) AS {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}},\n\ + \ ARRAY_AGG(TIMESTAMP({{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ ORDER BY {{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ AS {{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ \n \n \n FROM base_data\n GROUP\ + \ BY {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\n\ + \ " + taskInfo: + name: remove-feature-columns + bigquery-query-job-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job-2 + dependentTasks: + - build-job-configuration-query-2 + - get-table-location-2 + - table-to-uri-2 + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query-2 + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location-2 + pipelinechannel--table-to-uri-2-uri: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri-2 + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n WITH\n predictions AS (\n SELECT\n\ + \ {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}},\n\ + \ JSON_QUERY_ARRAY(prediction, '$.{{$.inputs.parameters['pipelinechannel--time_column']}}')\ + \ AS {{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ JSON_EXTRACT(\n prediction,\n \ + \ '$.predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}}'\n\ + \ ) AS predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ JSON_QUERY_ARRAY(\n prediction,\n \ + \ '$.predicted_{{$.inputs.parameters['pipelinechannel--target_column']}}'\n\ + \ ) AS predicted_{{$.inputs.parameters['pipelinechannel--target_column']}},\n\ + \ FROM `{{$.inputs.parameters['pipelinechannel--table-to-uri-2-uri']}}`\n\ + \ )\n SELECT\n {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}},\n\ + \ PARSE_TIMESTAMP(\n '\\\"%Y-%m-%dT%H:%M:%SZ\\\ + \"',\n predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}}\n\ + \ ) AS predicted_on_{{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ PARSE_TIMESTAMP(\n '\\\"%Y-%m-%dT%H:%M:%SZ\\\ + \"',\n {{$.inputs.parameters['pipelinechannel--time_column']}}[SAFE_OFFSET(index)]\n\ + \ ) AS {{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ STRUCT(\n CAST(predicted_{{$.inputs.parameters['pipelinechannel--target_column']}}[SAFE_OFFSET(index)]\ + \ AS FLOAT64)\n AS value\n ) AS predicted_{{$.inputs.parameters['pipelinechannel--target_column']}}\n\ + \ FROM predictions\n CROSS JOIN\n UNNEST(GENERATE_ARRAY(0,\ + \ ARRAY_LENGTH({{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ - 1)) AS index\n " + taskInfo: + name: create-predictions-table + build-job-configuration-query: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query + dependentTasks: + - bigquery-create-dataset + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}' + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}' + table_id: + runtimeValue: + constant: data + write_disposition: + runtimeValue: + constant: WRITE_EMPTY + taskInfo: + name: build-job-configuration-query + build-job-configuration-query-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query-2 + dependentTasks: + - table-to-uri-2 + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-2-dataset_id'']}}' + pipelinechannel--table-to-uri-2-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: table-to-uri-2 + pipelinechannel--table-to-uri-2-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: table-to-uri-2 + pipelinechannel--table-to-uri-2-table_id: + taskOutputParameter: + outputParameterKey: table_id + producerTask: table-to-uri-2 + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-2-project_id'']}}' + table_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--table-to-uri-2-table_id'']}}' + write_disposition: + runtimeValue: + constant: WRITE_TRUNCATE + taskInfo: + name: build-job-configuration-query-2 + get-first-valid: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-first-valid + dependentTasks: + - load-table-from-uri + inputs: + parameters: + pipelinechannel--data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + pipelinechannel--load-table-from-uri-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: load-table-from-uri + values: + runtimeValue: + constant: '["{{$.inputs.parameters[''pipelinechannel--data_source_bigquery_table_path'']}}", + "{{$.inputs.parameters[''pipelinechannel--load-table-from-uri-Output'']}}"]' + taskInfo: + name: get-first-valid + get-table-location: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-table-location + inputs: + parameters: + default_location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + table: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + taskInfo: + name: get-table-location + get-table-location-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-table-location-2 + dependentTasks: + - table-to-uri-2 + inputs: + parameters: + project: + componentInputParameter: pipelinechannel--project + table: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri-2 + taskInfo: + name: get-table-location-2 + load-table-from-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-load-table-from-uri + dependentTasks: + - bigquery-create-dataset + - get-table-location + inputs: + parameters: + destination: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}.{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}.csv_export' + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + project: + componentInputParameter: pipelinechannel--project + source_format: + runtimeValue: + constant: CSV + source_uris: + componentInputParameter: pipelinechannel--data_source_csv_filenames + taskInfo: + name: load-table-from-uri + make-vertex-model-artifact: + cachingOptions: + enableCache: true + componentRef: + name: comp-make-vertex-model-artifact + inputs: + parameters: + location: + componentInputParameter: pipelinechannel--location + model_resource_name: + componentInputParameter: pipelinechannel--model_name + taskInfo: + name: make-vertex-model-artifact + maybe-replace-with-default: + cachingOptions: + enableCache: true + componentRef: + name: comp-maybe-replace-with-default + inputs: + parameters: + default: + componentInputParameter: pipelinechannel--project + value: + componentInputParameter: pipelinechannel--bigquery_destination_uri + taskInfo: + name: maybe-replace-with-default + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + dependentTasks: + - make-vertex-model-artifact + - maybe-replace-with-default + - table-to-uri + inputs: + artifacts: + model: + taskOutputArtifact: + outputArtifactKey: vertex_model + producerTask: make-vertex-model-artifact + parameters: + bigquery_destination_output_uri: + runtimeValue: + constant: bq://{{$.inputs.parameters['pipelinechannel--maybe-replace-with-default-Output']}} + bigquery_source_input_uri: + runtimeValue: + constant: bq://{{$.inputs.parameters['pipelinechannel--table-to-uri-uri']}} + instances_format: + runtimeValue: + constant: bigquery + job_display_name: + runtimeValue: + constant: batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--machine_type + max_replica_count: + componentInputParameter: pipelinechannel--max_num_workers + pipelinechannel--maybe-replace-with-default-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: maybe-replace-with-default + pipelinechannel--table-to-uri-uri: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri + predictions_format: + runtimeValue: + constant: bigquery + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-batch-predict + table-to-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri + dependentTasks: + - bigquery-query-job + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: destination_table + producerTask: bigquery-query-job + taskInfo: + name: table-to-uri + table-to-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri-2 + dependentTasks: + - model-batch-predict + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: bigquery_output_table + producerTask: model-batch-predict + taskInfo: + name: table-to-uri-2 + validate-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-validate-inputs + inputs: + parameters: + bigquery_destination_uri: + componentInputParameter: pipelinechannel--bigquery_destination_uri + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--data_source_csv_filenames + taskInfo: + name: validate-inputs + inputDefinitions: + parameters: + pipelinechannel--bigquery_destination_uri: + parameterType: STRING + pipelinechannel--data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--data_source_csv_filenames: + parameterType: STRING + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--location: + parameterType: STRING + pipelinechannel--machine_type: + parameterType: STRING + pipelinechannel--max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--model_name: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_identifier_column: + parameterType: STRING + comp-get-first-valid: + executorLabel: exec-get-first-valid + inputDefinitions: + parameters: + values: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-table-location: + executorLabel: exec-get-table-location + inputDefinitions: + parameters: + default_location: + defaultValue: '' + description: Location to return if no table was given. + isOptional: true + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + table: + description: The BigQuery table to get a location for. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-table-location-2: + executorLabel: exec-get-table-location-2 + inputDefinitions: + parameters: + default_location: + defaultValue: '' + description: Location to return if no table was given. + isOptional: true + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + table: + description: The BigQuery table to get a location for. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-load-table-from-uri: + executorLabel: exec-load-table-from-uri + inputDefinitions: + parameters: + destination: + description: Table into which data is to be loaded. + parameterType: STRING + location: + description: The GCP region. + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + source_format: + defaultValue: CSV + description: 'The file format for the files being imported. Only CSV is + + supported.' + isOptional: true + parameterType: STRING + source_uris: + description: 'URIs of data files to be loaded; in format + + gs:///.' + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-make-vertex-model-artifact: + executorLabel: exec-make-vertex-model-artifact + inputDefinitions: + parameters: + location: + parameterType: STRING + model_resource_name: + parameterType: STRING + outputDefinitions: + artifacts: + vertex_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-maybe-replace-with-default: + executorLabel: exec-maybe-replace-with-default + inputDefinitions: + parameters: + default: + defaultValue: '' + isOptional: true + parameterType: STRING + value: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-table-to-uri: + executorLabel: exec-table-to-uri + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-table-to-uri-2: + executorLabel: exec-table-to-uri-2 + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-validate-inputs: + executorLabel: exec-validate-inputs + inputDefinitions: + parameters: + bigquery_destination_uri: + isOptional: true + parameterType: STRING + data_granularity_unit: + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + isOptional: true + parameterType: STRING + data_source_csv_filenames: + isOptional: true + parameterType: STRING + optimization_objective: + isOptional: true + parameterType: STRING + predefined_split_key: + isOptional: true + parameterType: STRING + source_model_uri: + isOptional: true + parameterType: STRING + target_column: + isOptional: true + parameterType: STRING + test_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + isOptional: true + parameterType: STRING + time_series_identifier_column: + isOptional: true + parameterType: STRING + timestamp_split_key: + isOptional: true + parameterType: STRING + training_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + window_column: + isOptional: true + parameterType: STRING + window_max_count: + isOptional: true + parameterType: NUMBER_INTEGER + window_stride_length: + isOptional: true + parameterType: NUMBER_INTEGER +deploymentSpec: + executors: + exec-bigquery-create-dataset: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_create_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_create_dataset(\n project: str,\n location: str,\n\ + \ dataset: str,\n exists_ok: bool = False,\n) -> NamedTuple('Outputs',\ + \ [('project_id', str), ('dataset_id', str)]):\n \"\"\"Creates a BigQuery\ + \ dataset.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n\n from google.cloud import bigquery\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n ref\ + \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ + \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ + \ ref.project, ref.dataset_id)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-delete-dataset-with-prefix: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_delete_dataset_with_prefix + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_delete_dataset_with_prefix(\n project: str,\n \ + \ dataset_prefix: str,\n delete_contents: bool = False,\n) -> None:\n\ + \ \"\"\"Deletes all BigQuery datasets matching the given prefix.\"\"\"\n\ + \ # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project)\n for dataset in client.list_datasets(project=project):\n\ + \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ + \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-query-job: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-bigquery-query-job-2: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-build-job-configuration-query: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-build-job-configuration-query-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-first-valid: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_first_valid + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_first_valid(values: str) -> str:\n \"\"\"Returns the first\ + \ truthy value from the given serialized JSON list.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n for value in json.loads(values):\n if value:\n return value\n\ + \ raise ValueError('No valid values.')\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-table-location: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_table_location + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_table_location(\n project: str,\n table: Optional[str],\n\ + \ default_location: str = '',\n) -> str:\n \"\"\"Returns the region\ + \ the given table belongs to.\n\n Args:\n project: The GCP project.\n\ + \ table: The BigQuery table to get a location for.\n default_location:\ + \ Location to return if no table was given.\n\n Returns:\n A GCP region\ + \ or multi-region.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n if not table:\n return default_location\n\n client = bigquery.Client(project=project)\n\ + \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ + \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ + \ return client.get_table(table).location\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-table-location-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_table_location + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_table_location(\n project: str,\n table: Optional[str],\n\ + \ default_location: str = '',\n) -> str:\n \"\"\"Returns the region\ + \ the given table belongs to.\n\n Args:\n project: The GCP project.\n\ + \ table: The BigQuery table to get a location for.\n default_location:\ + \ Location to return if no table was given.\n\n Returns:\n A GCP region\ + \ or multi-region.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n if not table:\n return default_location\n\n client = bigquery.Client(project=project)\n\ + \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ + \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ + \ return client.get_table(table).location\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-load-table-from-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - load_table_from_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef load_table_from_uri(\n project: str,\n location: str,\n\ + \ source_uris: str,\n destination: str,\n source_format: str =\ + \ 'CSV',\n) -> str:\n \"\"\"Creates a table from a list of URIs.\n\n Args:\n\ + \ project: The GCP project.\n location: The GCP region.\n source_uris:\ + \ URIs of data files to be loaded; in format\n gs:///.\n\ + \ destination: Table into which data is to be loaded.\n source_format:\ + \ The file format for the files being imported. Only CSV is\n supported.\n\ + \n Returns:\n The destination table containing imported data.\n \"\"\ + \"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n if not source_uris:\n return ''\n\n csv_list = [filename.strip()\ + \ for filename in source_uris.split(',')]\n client = bigquery.Client(project=project,\ + \ location=location)\n job_config = bigquery.LoadJobConfig(\n autodetect=True,\ + \ source_format=source_format)\n client.load_table_from_uri(\n source_uris=csv_list,\n\ + \ destination=destination,\n project=project,\n location=location,\n\ + \ job_config=job_config).result()\n return destination\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-make-vertex-model-artifact: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - make_vertex_model_artifact + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef make_vertex_model_artifact(\n location: str,\n model_resource_name:\ + \ str,\n vertex_model: dsl.Output[dsl.Artifact],\n) -> None:\n \"\"\"\ + Creates a google.VertexModel artifact.\"\"\"\n vertex_model.metadata =\ + \ {'resourceName': model_resource_name}\n vertex_model.uri = (f'https://{location}-aiplatform.googleapis.com'\n\ + \ f'/v1/{model_resource_name}')\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-maybe-replace-with-default: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - maybe_replace_with_default + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef maybe_replace_with_default(value: str, default: str = '') ->\ + \ str:\n \"\"\"Replaces string with another value if it is a dash.\"\"\"\ + \n return default if not value else value\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-table-to-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-table-to-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-validate-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - validate_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef validate_inputs(\n time_column: Optional[str] = None,\n \ + \ time_series_identifier_column: Optional[str] = None,\n target_column:\ + \ Optional[str] = None,\n data_source_bigquery_table_path: Optional[str]\ + \ = None,\n training_fraction: Optional[float] = None,\n validation_fraction:\ + \ Optional[float] = None,\n test_fraction: Optional[float] = None,\n\ + \ predefined_split_key: Optional[str] = None,\n timestamp_split_key:\ + \ Optional[str] = None,\n data_source_csv_filenames: Optional[str] =\ + \ None,\n source_model_uri: Optional[str] = None,\n bigquery_destination_uri:\ + \ Optional[str] = None,\n window_column: Optional[str] = None,\n window_stride_length:\ + \ Optional[int] = None,\n window_max_count: Optional[int] = None,\n \ + \ optimization_objective: Optional[str] = None,\n data_granularity_unit:\ + \ Optional[str] = None,\n) -> None:\n \"\"\"Checks training pipeline input\ + \ parameters are valid.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import re\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n project_pattern = r'([a-z0-9.-]+:)?[a-z][a-z0-9-_]{4,28}[a-z0-9]'\n\ + \ dataset_pattern = r'[a-zA-Z0-9_]+'\n table_pattern = r'[^\\.\\:`]+'\n\ + \ dataset_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}')\n\ + \ table_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}[.:]{table_pattern}')\n\ + \n # Validate BigQuery column and dataset names.\n bigquery_column_parameters\ + \ = [\n time_column,\n time_series_identifier_column,\n target_column,\n\ + \ ]\n column_pattern = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]{1,300}')\n \ + \ for column in bigquery_column_parameters:\n if column and not column_pattern.fullmatch(column):\n\ + \ raise ValueError(f'Invalid column name: {column}.')\n if (bigquery_destination_uri\ + \ and\n not dataset_uri_pattern.fullmatch(bigquery_destination_uri)):\n\ + \ raise ValueError(\n f'Invalid BigQuery dataset URI: {bigquery_destination_uri}.')\n\ + \ if (source_model_uri and not table_uri_pattern.fullmatch(source_model_uri)):\n\ + \ raise ValueError(f'Invalid BigQuery table URI: {source_model_uri}.')\n\ + \n # Validate data source.\n data_source_count = sum([bool(source) for\ + \ source in [\n data_source_bigquery_table_path, data_source_csv_filenames]])\n\ + \ if data_source_count > 1:\n raise ValueError(f'Expected 1 data source,\ + \ found {data_source_count}.')\n if (data_source_bigquery_table_path\n\ + \ and not table_uri_pattern.fullmatch(data_source_bigquery_table_path)):\n\ + \ raise ValueError(\n f'Invalid BigQuery table URI: {data_source_bigquery_table_path}.')\n\ + \ gcs_path_pattern = re.compile(r'gs:\\/\\/(.+)\\/([^\\/]+)')\n if data_source_csv_filenames:\n\ + \ csv_list = [filename.strip()\n for filename in data_source_csv_filenames.split(',')]\n\ + \ for gcs_path in csv_list:\n if not gcs_path_pattern.fullmatch(gcs_path):\n\ + \ raise ValueError(f'Invalid path to CSV stored in GCS: {gcs_path}.')\n\ + \n # Validate split spec.\n fraction_splits = [\n training_fraction,\n\ + \ validation_fraction,\n test_fraction,\n ]\n fraction_splits\ + \ = [None if fraction == -1 else fraction\n for fraction\ + \ in fraction_splits]\n split_count = sum([\n bool(source)\n \ + \ for source in [predefined_split_key,\n any(fraction_splits)]\n\ + \ ])\n if split_count > 1:\n raise ValueError(f'Expected 1 split type,\ + \ found {split_count}.')\n if (predefined_split_key and\n not column_pattern.fullmatch(predefined_split_key)):\n\ + \ raise ValueError(f'Invalid column name: {predefined_split_key}.')\n\ + \ if any(fraction_splits):\n if not all(fraction_splits):\n raise\ + \ ValueError(\n f'All fractions must be non-zero. Got: {fraction_splits}.')\n\ + \ if sum(fraction_splits) != 1:\n raise ValueError(\n f'Fraction\ + \ splits must sum to 1. Got: {sum(fraction_splits)}.')\n if (timestamp_split_key\ + \ and\n not column_pattern.fullmatch(timestamp_split_key)):\n raise\ + \ ValueError(f'Invalid column name: {timestamp_split_key}.')\n if timestamp_split_key\ + \ and not all(fraction_splits):\n raise ValueError('All fractions must\ + \ be non-zero for timestamp split.')\n\n # Validate window config.\n if\ + \ window_stride_length == -1:\n window_stride_length = None\n if window_max_count\ + \ == -1:\n window_max_count = None\n window_configs = [window_column,\ + \ window_stride_length, window_max_count]\n window_config_count = sum([bool(config)\ + \ for config in window_configs])\n if window_config_count > 1:\n raise\ + \ ValueError(f'Expected 1 window config, found {window_config_count}.')\n\ + \ if window_column and not column_pattern.fullmatch(window_column):\n \ + \ raise ValueError(f'Invalid column name: {window_column}.')\n if window_stride_length\ + \ and (window_stride_length < 1 or\n window_stride_length\ + \ > 1000):\n raise ValueError('Stride must be between 1 and 1000. Got:\ + \ '\n f'{window_stride_length}.')\n if window_max_count\ + \ and (window_max_count < 1000 or\n window_max_count\ + \ > int(1e8)):\n raise ValueError('Max count must be between 1000 and\ + \ 100000000. Got: '\n f'{window_max_count}.')\n\n #\ + \ Validate eval metric.\n valid_optimization_objectives = ['rmse', 'mae',\ + \ 'rmsle']\n if optimization_objective:\n if optimization_objective\ + \ not in valid_optimization_objectives:\n raise ValueError(\n \ + \ 'Optimization objective should be one of the following: '\n \ + \ f'{valid_optimization_objectives}, got: {optimization_objective}.')\n\ + \n # Validate data granularity unit.\n valid_data_granularity_units =\ + \ [\n 'minute', 'hour', 'day', 'week', 'month', 'year']\n if data_granularity_unit:\n\ + \ if data_granularity_unit not in valid_data_granularity_units:\n \ + \ raise ValueError(\n 'Granularity unit should be one of the\ + \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 +pipelineInfo: + description: Creates a batch prediction using a Prophet model. + name: prophet-predict +root: + dag: + tasks: + bigquery-delete-dataset-with-prefix: + cachingOptions: {} + componentRef: + name: comp-bigquery-delete-dataset-with-prefix + dependentTasks: + - exit-handler-1 + inputs: + parameters: + dataset_prefix: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + delete_contents: + runtimeValue: + constant: true + project: + componentInputParameter: project + taskInfo: + name: delete-tmp-dataset + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + inputs: + parameters: + pipelinechannel--bigquery_destination_uri: + componentInputParameter: bigquery_destination_uri + pipelinechannel--data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + pipelinechannel--data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--machine_type: + componentInputParameter: machine_type + pipelinechannel--max_num_workers: + componentInputParameter: max_num_workers + pipelinechannel--model_name: + componentInputParameter: model_name + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--time_column: + componentInputParameter: time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: time_series_identifier_column + taskInfo: + name: exit-handler-1 + inputDefinitions: + parameters: + bigquery_destination_uri: + defaultValue: '' + description: 'URI of the desired destination dataset. If not + + specified, resources will be created under a new dataset in the project. + + Unlike in Vertex Forecasting, all resources will be given hardcoded names + + under this dataset, and the model artifact will also be exported here.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + location: + description: The GCP region for Vertex AI. + parameterType: STRING + machine_type: + defaultValue: n1-standard-2 + description: The machine type used for batch prediction. + isOptional: true + parameterType: STRING + max_num_workers: + defaultValue: 10.0 + description: The max number of workers used for batch prediction. + isOptional: true + parameterType: NUMBER_INTEGER + model_name: + description: 'The name of the Model resource, in a form of + + projects/{project}/locations/{location}/models/{model}.' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + target_column: + description: Name of the column that the model is to predict values for. + parameterType: STRING + time_column: + description: 'Name of the column that identifies time order in the time + + series.' + parameterType: STRING + time_series_identifier_column: + description: 'Name of the column that identifies the time + + series.' + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py new file mode 100644 index 0000000000..c79a97c815 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer.py @@ -0,0 +1,190 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prophet trainer component spec.""" + +from typing import Optional +from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Output + + +# pylint: disable=g-doc-args,unused-argument +@dsl.container_component +def prophet_trainer( + project: str, + location: str, + root_dir: str, + target_column: str, + time_column: str, + time_series_identifier_column: str, + forecast_horizon: int, + window_column: str, + data_granularity_unit: str, + predefined_split_column: str, + source_bigquery_uri: str, + gcp_resources: dsl.OutputPath(str), + unmanaged_container_model: Output[UnmanagedContainerModel], + evaluated_examples_directory: Output[Artifact], + optimization_objective: Optional[str] = 'rmse', + max_num_trials: Optional[int] = 6, + encryption_spec_key_name: Optional[str] = '', + dataflow_max_num_workers: Optional[int] = 10, + dataflow_machine_type: Optional[str] = 'n1-standard-1', + dataflow_disk_size_gb: Optional[int] = 40, + dataflow_service_account: Optional[str] = '', + dataflow_subnetwork: Optional[str] = '', + dataflow_use_public_ips: Optional[bool] = True, +): + # fmt: off + """Trains and tunes one Prophet model per time series using Dataflow. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region for Vertex AI. + root_dir: The Cloud Storage location to store the output. + time_column: Name of the column that identifies time order in the time series. + time_series_identifier_column: Name of the column that identifies the time series. + target_column: Name of the column that the model is to predict values for. + forecast_horizon: The number of time periods into the future for which forecasts will be created. Future periods start after the latest timestamp for each time series. + optimization_objective: Optimization objective for tuning. Supported metrics come from Prophet's performance_metrics function. These are mse, rmse, mae, mape, mdape, smape, and coverage. + data_granularity_unit: String representing the units of time for the time column. + predefined_split_column: The predefined_split column name. A string that represents a list of comma separated CSV filenames. + source_bigquery_uri: The BigQuery table path of format bq (str)://bq_project.bq_dataset.bq_table + window_column: Name of the column that should be used to filter input rows. The column should contain either booleans or string booleans; if the value of the row is True, generate a sliding window from that row. + max_num_trials: Maximum number of tuning trials to perform per time series. There are up to 100 possible combinations to explore for each time series. Recommended values to try are 3, 6, and 24. + encryption_spec_key_name: Customer-managed encryption key. + dataflow_machine_type: The dataflow machine type used for training. + dataflow_max_num_workers: The max number of Dataflow workers used for training. + dataflow_disk_size_gb: Dataflow worker's disk size in GB during training. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + + Returns: + gcp_resources: Serialized gcp_resources proto tracking the custom training job. + unmanaged_container_model: The UnmanagedContainerModel artifact. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + '{"display_name": ' + + f'"prophet-trainer-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}", ', + '"encryption_spec": {"kms_key_name":"', + encryption_spec_key_name, + '"}, ', + '"job_spec": {"worker_pool_specs": [{"replica_count":"1", ', + '"machine_spec": {"machine_type": "n1-standard-4"}, ', + ( + '"container_spec":' + ' {"image_uri":"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", ' + ), + '"args": ["prophet_trainer", "', + ( + f'--job_name=dataflow-{dsl.PIPELINE_JOB_NAME_PLACEHOLDER}", "' + ), + ( + '--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", "' + ), + ( + '--prediction_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/fte-prediction-server:20231002_0125", "' + ), + '--artifacts_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/model/", "', + '--evaluated_examples_dir=', + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/eval/", "', + '--region=', + location, + '", "', + '--source_bigquery_uri=', + source_bigquery_uri, + '", "', + '--target_column=', + target_column, + '", "', + '--time_column=', + time_column, + '", "', + '--time_series_identifier_column=', + time_series_identifier_column, + '", "', + '--forecast_horizon=', + forecast_horizon, + '", "', + '--window_column=', + window_column, + '", "', + '--optimization_objective=', + optimization_objective, + '", "', + '--data_granularity_unit=', + data_granularity_unit, + '", "', + '--predefined_split_column=', + predefined_split_column, + '", "', + '--max_num_trials=', + max_num_trials, + '", "', + '--dataflow_project=', + project, + '", "', + '--dataflow_max_num_workers=', + dataflow_max_num_workers, + '", "', + '--dataflow_machine_type=', + dataflow_machine_type, + '", "', + '--dataflow_disk_size_gb=', + dataflow_disk_size_gb, + '", "', + '--dataflow_service_account=', + dataflow_service_account, + '", "', + '--dataflow_subnetwork=', + dataflow_subnetwork, + '", "', + '--dataflow_use_public_ips=', + dataflow_use_public_ips, + '", "', + '--gcp_resources_path=', + gcp_resources, + '", "', + '--executor_input={{$.json_escape[1]}}"]}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml new file mode 100644 index 0000000000..e0e311d4ef --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/prophet_trainer_pipeline.yaml @@ -0,0 +1,3050 @@ +# PIPELINE DEFINITION +# Name: prophet-train +# Description: Trains one Prophet model per time series. +# Inputs: +# data_granularity_unit: str +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# encryption_spec_key_name: str [Default: ''] +# evaluation_dataflow_disk_size_gb: int [Default: 40.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-1'] +# evaluation_dataflow_max_num_workers: int [Default: 10.0] +# forecast_horizon: int +# location: str +# max_num_trials: int [Default: 6.0] +# optimization_objective: str +# predefined_split_key: str [Default: ''] +# project: str +# root_dir: str +# run_evaluation: bool [Default: True] +# target_column: str +# test_fraction: float [Default: -1.0] +# time_column: str +# time_series_identifier_column: str +# timestamp_split_key: str [Default: ''] +# trainer_dataflow_disk_size_gb: int [Default: 40.0] +# trainer_dataflow_machine_type: str [Default: 'n1-standard-1'] +# trainer_dataflow_max_num_workers: int [Default: 10.0] +# training_fraction: float [Default: -1.0] +# validation_fraction: float [Default: -1.0] +# window_column: str [Default: ''] +# window_max_count: int [Default: -1.0] +# window_stride_length: int [Default: -1.0] +components: + comp-bigquery-create-dataset: + executorLabel: exec-bigquery-create-dataset + inputDefinitions: + parameters: + dataset: + parameterType: STRING + exists_ok: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + comp-bigquery-delete-dataset-with-prefix: + executorLabel: exec-bigquery-delete-dataset-with-prefix + inputDefinitions: + parameters: + dataset_prefix: + parameterType: STRING + delete_contents: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + project: + parameterType: STRING + comp-bigquery-query-job: + executorLabel: exec-bigquery-query-job + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: 'Describes the Cloud + + KMS encryption key that will be used to protect destination + + BigQuery table. The BigQuery Service Account associated with your + + project requires access to this encryption key. If + + encryption_spec_key_name are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + job_configuration_query: + defaultValue: {} + description: 'A json formatted string + + describing the rest of the job configuration. For more details, see + + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels associated with this job. You can + + use these to organize and group your jobs. Label keys and values can + + be no longer than 63 characters, can only containlowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. Label values are optional. Label keys must start with a + + letter and each label in the list must have a different key. + + Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Location for creating the BigQuery job. If not + + set, default to `US` multi-region. For more details, see + + https://cloud.google.com/bigquery/docs/locations#specifying_your_location' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run the BigQuery query job. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + query: + defaultValue: '' + description: 'SQL query text to execute. Only standard SQL is + + supported. If query are both specified in here and in + + job_configuration_query, the value in here will override the other + + one.' + isOptional: true + parameterType: STRING + query_parameters: + defaultValue: [] + description: 'jobs.query parameters for + + standard SQL queries. If query_parameters are both specified in here + + and in job_configuration_query, the value in here will override the + + other one.' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + destination_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Describes the table where the query results should be stored. + + This property must be set for large results that exceed the maximum + + response size. + + For queries that produce anonymous (cached) results, this field will + + be populated by BigQuery.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the BigQuery job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-build-job-configuration-query: + executorLabel: exec-build-job-configuration-query + inputDefinitions: + parameters: + dataset_id: + defaultValue: '' + isOptional: true + parameterType: STRING + priority: + defaultValue: INTERACTIVE + isOptional: true + parameterType: STRING + project_id: + defaultValue: '' + isOptional: true + parameterType: STRING + table_id: + defaultValue: '' + isOptional: true + parameterType: STRING + write_disposition: + defaultValue: '' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRUCT + comp-condition-2: + dag: + tasks: + model-evaluation-regression: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-regression + inputs: + artifacts: + predictions_gcs_source: + componentInputArtifact: pipelinechannel--prophet-trainer-evaluated_examples_directory + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_gcs_source: + runtimeValue: + constant: [] + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + prediction_score_column: + runtimeValue: + constant: prediction.predicted_{{$.inputs.parameters['pipelinechannel--target_column']}} + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + target_field_name: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: model-evaluation-regression + inputDefinitions: + artifacts: + pipelinechannel--prophet-trainer-evaluated_examples_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--target_column: + parameterType: STRING + comp-exit-handler-1: + dag: + tasks: + bigquery-create-dataset: + cachingOptions: {} + componentRef: + name: comp-bigquery-create-dataset + dependentTasks: + - get-table-location + - validate-inputs + inputs: + parameters: + dataset: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: create-tmp-dataset + bigquery-query-job: + cachingOptions: + enableCache: true + componentRef: + name: comp-bigquery-query-job + dependentTasks: + - bigquery-create-dataset + - build-job-configuration-query + - get-fte-suffix + - get-table-location + inputs: + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + job_configuration_query: + taskOutputParameter: + outputParameterKey: Output + producerTask: build-job-configuration-query + location: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-table-location + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + pipelinechannel--get-fte-suffix-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-fte-suffix + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--time_column: + componentInputParameter: pipelinechannel--time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + project: + componentInputParameter: pipelinechannel--project + query: + runtimeValue: + constant: "\n WITH\n base_data AS (\n SELECT\ + \ * FROM `{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-project_id']}}.{{$.inputs.parameters['pipelinechannel--bigquery-create-dataset-dataset_id']}}.fte_time_series_output_{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}`\n\ + \ )\n SELECT\n CAST({{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\ + \ AS STRING) AS {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}},\n\ + \ ARRAY_AGG(TIMESTAMP({{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ ORDER BY {{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ AS {{$.inputs.parameters['pipelinechannel--time_column']}},\n\ + \ ARRAY_AGG({{$.inputs.parameters['pipelinechannel--target_column']}}\ + \ ORDER BY {{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ AS {{$.inputs.parameters['pipelinechannel--target_column']}},\n\ + \ ARRAY_AGG(split__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}\ + \ ORDER BY {{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ AS split__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}},\n\ + \ ARRAY_AGG(window__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}}\ + \ ORDER BY {{$.inputs.parameters['pipelinechannel--time_column']}})\ + \ AS window__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}},\n\ + \ FROM base_data\n GROUP BY {{$.inputs.parameters['pipelinechannel--time_series_identifier_column']}}\n\ + \ " + taskInfo: + name: aggregate-by-time-series-id + build-job-configuration-query: + cachingOptions: + enableCache: true + componentRef: + name: comp-build-job-configuration-query + dependentTasks: + - bigquery-create-dataset + inputs: + parameters: + dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}' + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + project_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}' + table_id: + runtimeValue: + constant: data + write_disposition: + runtimeValue: + constant: WRITE_EMPTY + taskInfo: + name: build-job-configuration-query + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - prophet-trainer + inputs: + artifacts: + pipelinechannel--prophet-trainer-evaluated_examples_directory: + taskOutputArtifact: + outputArtifactKey: evaluated_examples_directory + producerTask: prophet-trainer + parameters: + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: run-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--run_evaluation'] + == true + feature-transform-engine: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-transform-engine + dependentTasks: + - bigquery-create-dataset + - wrapped-in-list + inputs: + parameters: + autodetect_csv_schema: + runtimeValue: + constant: true + bigquery_staging_full_dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}.{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}' + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--data_source_csv_filenames + forecasting_apply_windowing: + runtimeValue: + constant: false + forecasting_context_window: + runtimeValue: + constant: 0.0 + forecasting_forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + forecasting_predefined_window_column: + componentInputParameter: pipelinechannel--window_column + forecasting_time_column: + componentInputParameter: pipelinechannel--time_column + forecasting_time_series_identifier_columns: + taskOutputParameter: + outputParameterKey: Output + producerTask: wrapped-in-list + forecasting_window_max_count: + componentInputParameter: pipelinechannel--window_max_count + forecasting_window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + runtimeValue: + constant: time_series + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + tf_auto_transform_features: + runtimeValue: + constant: {} + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + taskInfo: + name: feature-transform-engine + get-fte-suffix: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-fte-suffix + dependentTasks: + - bigquery-create-dataset + - feature-transform-engine + inputs: + parameters: + bigquery_staging_full_dataset_id: + runtimeValue: + constant: '{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-project_id'']}}.{{$.inputs.parameters[''pipelinechannel--bigquery-create-dataset-dataset_id'']}}' + fte_table: + runtimeValue: + constant: fte_time_series_output + location: + componentInputParameter: pipelinechannel--location + pipelinechannel--bigquery-create-dataset-dataset_id: + taskOutputParameter: + outputParameterKey: dataset_id + producerTask: bigquery-create-dataset + pipelinechannel--bigquery-create-dataset-project_id: + taskOutputParameter: + outputParameterKey: project_id + producerTask: bigquery-create-dataset + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: get-fte-suffix + get-table-location: + cachingOptions: + enableCache: true + componentRef: + name: comp-get-table-location + inputs: + parameters: + default_location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + table: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + taskInfo: + name: get-table-location + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - prophet-trainer + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: prophet-trainer + parameters: + description: + runtimeValue: + constant: Prophet model. + display_name: + runtimeValue: + constant: prophet_{{$.pipeline_job_uuid}} + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + prophet-trainer: + cachingOptions: + enableCache: true + componentRef: + name: comp-prophet-trainer + dependentTasks: + - get-fte-suffix + - table-to-uri + inputs: + parameters: + data_granularity_unit: + componentInputParameter: pipelinechannel--data_granularity_unit + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--trainer_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--trainer_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--trainer_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + forecast_horizon: + componentInputParameter: pipelinechannel--forecast_horizon + location: + componentInputParameter: pipelinechannel--location + max_num_trials: + componentInputParameter: pipelinechannel--max_num_trials + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + pipelinechannel--get-fte-suffix-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: get-fte-suffix + pipelinechannel--table-to-uri-uri: + taskOutputParameter: + outputParameterKey: uri + producerTask: table-to-uri + predefined_split_column: + runtimeValue: + constant: split__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}} + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + source_bigquery_uri: + runtimeValue: + constant: bq://{{$.inputs.parameters['pipelinechannel--table-to-uri-uri']}} + target_column: + componentInputParameter: pipelinechannel--target_column + time_column: + componentInputParameter: pipelinechannel--time_column + time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + window_column: + runtimeValue: + constant: window__{{$.inputs.parameters['pipelinechannel--get-fte-suffix-Output']}} + taskInfo: + name: prophet-trainer + table-to-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-table-to-uri + dependentTasks: + - bigquery-query-job + inputs: + artifacts: + table: + taskOutputArtifact: + outputArtifactKey: destination_table + producerTask: bigquery-query-job + taskInfo: + name: table-to-uri + validate-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-validate-inputs + inputs: + parameters: + data_granularity_unit: + componentInputParameter: pipelinechannel--data_granularity_unit + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--data_source_csv_filenames + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + target_column: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + time_column: + componentInputParameter: pipelinechannel--time_column + time_series_identifier_column: + componentInputParameter: pipelinechannel--time_series_identifier_column + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + window_column: + componentInputParameter: pipelinechannel--window_column + window_max_count: + componentInputParameter: pipelinechannel--window_max_count + window_stride_length: + componentInputParameter: pipelinechannel--window_stride_length + taskInfo: + name: validate-inputs + wrapped-in-list: + cachingOptions: + enableCache: true + componentRef: + name: comp-wrapped-in-list + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--time_series_identifier_column + taskInfo: + name: wrapped-in-list + inputDefinitions: + parameters: + pipelinechannel--data_granularity_unit: + parameterType: STRING + pipelinechannel--data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--data_source_csv_filenames: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--forecast_horizon: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--max_num_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--time_column: + parameterType: STRING + pipelinechannel--time_series_identifier_column: + parameterType: STRING + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--trainer_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--trainer_dataflow_machine_type: + parameterType: STRING + pipelinechannel--trainer_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--window_column: + parameterType: STRING + pipelinechannel--window_max_count: + parameterType: NUMBER_INTEGER + pipelinechannel--window_stride_length: + parameterType: NUMBER_INTEGER + comp-feature-transform-engine: + executorLabel: exec-feature-transform-engine + inputDefinitions: + parameters: + autodetect_csv_schema: + defaultValue: false + description: 'If True, infers the column types + + when importing CSVs into BigQuery.' + isOptional: true + parameterType: BOOLEAN + bigquery_staging_full_dataset_id: + defaultValue: '' + description: 'Dataset in + + "projectId.datasetId" format for storing intermediate-FTE BigQuery + + tables. If the specified dataset does not exist in BigQuery, FTE will + + create the dataset. If no bigquery_staging_full_dataset_id is specified, + + all intermediate tables will be stored in a dataset created under the + + provided project in the input data source''s location during FTE + + execution called + + "vertex_feature_transform_engine_staging_{location.replace(''-'', ''_'')}". + + All tables generated by FTE will have a 30 day TTL.' + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'BigQuery input data + + source to run feature transform on.' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'CSV input data source to run + + feature transform on.' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + Dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataset_level_custom_transformation_definitions: + defaultValue: [] + description: "List of dataset-level custom transformation definitions. \ + \ Custom,\nbring-your-own dataset-level transform functions, where users\ + \ can define\nand import their own transform function and use it with\ + \ FTE's built-in\ntransformations. Using custom transformations is an\ + \ experimental feature\nand it is currently not supported during batch\ + \ prediction.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"ConcatCols\",\n \"module_path\": \"/path/to/custom_transform_fn_dlt.py\"\ + ,\n \"function_name\": \"concat_cols\" } ] Using custom transform\ + \ function\n together with FTE's built-in transformations: .. code-block::\n\ + \ python [ { \"transformation\": \"Join\", \"right_table_uri\":\n\ + \ \"bq://test-project.dataset_test.table\", \"join_keys\":\n [[\"\ + join_key_col\", \"join_key_col\"]] },{ \"transformation\":\n \"ConcatCols\"\ + , \"cols\": [\"feature_1\", \"feature_2\"], \"output_col\":\n \"feature_1_2\"\ + \ } ]" + isOptional: true + parameterType: LIST + dataset_level_transformations: + defaultValue: [] + description: "List of dataset-level\ntransformations.\nExample: .. code-block::\ + \ python [ { \"transformation\": \"Join\",\n \"right_table_uri\": \"\ + bq://test-project.dataset_test.table\",\n \"join_keys\": [[\"join_key_col\"\ + , \"join_key_col\"]] }, ... ] Additional\n information about FTE's currently\ + \ supported built-in\n transformations:\n Join: Joins features from\ + \ right_table_uri. For each join key, the\n left table keys will\ + \ be included and the right table keys will\n be dropped.\n \ + \ Example: .. code-block:: python { \"transformation\": \"Join\",\n\ + \ \"right_table_uri\": \"bq://test-project.dataset_test.table\"\ + ,\n \"join_keys\": [[\"join_key_col\", \"join_key_col\"]] }\n\ + \ Arguments:\n right_table_uri: Right table BigQuery\ + \ uri to join\n with input_full_table_id.\n join_keys:\ + \ Features to join on. For each\n nested list, the first\ + \ element is a left table column\n and the second is its\ + \ corresponding right table column.\n TimeAggregate: Creates a new\ + \ feature composed of values of an\n existing feature from a fixed\ + \ time period ago or in the future.\n Ex: A feature for sales by\ + \ store 1 year ago.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"TimeAggregate\", \"time_difference\": 40,\n \"\ + time_difference_units\": \"DAY\",\n \"time_series_identifier_columns\"\ + : [\"store_id\"],\n \"time_column\": \"time_col\", \"time_difference_target_column\"\ + :\n \"target_col\", \"output_column\": \"output_col\" }\n \ + \ Arguments:\n time_difference: Number of time_difference_units\ + \ to\n look back or into the future on our\n \ + \ time_difference_target_column.\n time_difference_units:\ + \ Units of time_difference to\n look back or into the future\ + \ on our\n time_difference_target_column. Must be one of\ + \ * 'DAY' *\n 'WEEK' (Equivalent to 7 DAYs) * 'MONTH' * 'QUARTER'\ + \ *\n 'YEAR'\n time_series_identifier_columns:\ + \ Names of the\n time series identifier columns.\n \ + \ time_column: Name of the time column.\n time_difference_target_column:\ + \ Column we wish to get\n the value of time_difference time_difference_units\ + \ in\n the past or future.\n output_column: Name\ + \ of our new time aggregate\n feature.\n is_future:\ + \ Whether we wish to look\n forward in time. Defaults to\ + \ False.\n PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:\n\ + \ Performs a partition by reduce operation (one of max,\n\ + \ min, avg, or sum) with a fixed historic time period. Ex:\n\ + \ Getting avg sales (the reduce column) for each store\n\ + \ (partition_by_column) over the previous 5 days\n \ + \ (time_column, time_ago_units, and time_ago).\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"PartitionByMax\"\ + , \"reduce_column\": \"sell_price\",\n \"partition_by_columns\"\ + : [\"store_id\", \"state_id\"],\n \"time_column\": \"date\",\ + \ \"time_ago\": 1, \"time_ago_units\":\n \"WEEK\", \"output_column\"\ + : \"partition_by_reduce_max_output\" }\n Arguments:\n \ + \ reduce_column: Column to apply the reduce operation\n \ + \ on. Reduce operations include the\n following: Max,\ + \ Min, Avg, Sum.\n partition_by_columns: List of columns to\n\ + \ partition by.\n time_column: Time column for\ + \ the partition by\n operation's window function.\n \ + \ time_ago: Number of time_ago_units to look back on\n \ + \ our target_column, starting from time_column\n (inclusive).\n\ + \ time_ago_units: Units of time_ago to look back on\n \ + \ our target_column. Must be one of * 'DAY' * 'WEEK'\n \ + \ output_column: Name of our output feature." + isOptional: true + parameterType: LIST + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + feature_selection_algorithm: + defaultValue: AMI + description: "The algorithm of feature\nselection. One of \"AMI\", \"CMIM\"\ + , \"JMIM\", \"MRMR\", default to be \"AMI\".\nThe algorithms available\ + \ are: AMI(Adjusted Mutual Information):\n Reference:\n https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html\n\ + \ Arrays are not yet supported in this algorithm. CMIM(Conditional\n\ + \ Mutual Information Maximization): Reference paper: Mohamed\n \ + \ Bennasar, Yulia Hicks, Rossitza Setchi, \u201CFeature selection\ + \ using\n Joint Mutual Information Maximisation,\u201D Expert Systems\ + \ with\n Applications, vol. 42, issue 22, 1 December 2015, Pages\n\ + \ 8520-8532. JMIM(Joint Mutual Information Maximization): Reference\n\ + \ paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, \u201C\ + Feature\n selection using Joint Mutual Information Maximisation,\u201D\ + \ Expert\n Systems with Applications, vol. 42, issue 22, 1 December\ + \ 2015,\n Pages 8520-8532. MRMR(MIQ Minimum-redundancy\n \ + \ Maximum-relevance): Reference paper: Hanchuan Peng, Fuhui Long,\n\ + \ and Chris Ding. \"Feature selection based on mutual information\n\ + \ criteria of max-dependency, max-relevance, and min-redundancy.\"\ + \n IEEE Transactions on pattern analysis and machine intelligence\n\ + \ 27, no.\n 8: 1226-1238." + isOptional: true + parameterType: STRING + feature_selection_execution_engine: + defaultValue: dataflow + description: Execution engine to run feature selection, value can be dataflow, + bigquery. + isOptional: true + parameterType: STRING + forecasting_apply_windowing: + defaultValue: true + description: Whether to apply window strategy. + isOptional: true + parameterType: BOOLEAN + forecasting_available_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + available at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_context_window: + defaultValue: -1.0 + description: Forecasting context window. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_forecast_horizon: + defaultValue: -1.0 + description: Forecasting horizon. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_holiday_regions: + defaultValue: [] + description: 'The geographical region based on which the + + holiday effect is applied in modeling by adding holiday categorical + + array feature that include all holidays matching the date. This option + + only allowed when data granularity is day. By default, holiday effect + + modeling is disabled. To turn it on, specify the holiday region using + + this option. + + Top level: * ''GLOBAL'' + + Second level: continental regions: * ''NA'': North America + + * ''JAPAC'': Japan and Asia Pacific + + * ''EMEA'': Europe, the Middle East and Africa + + * ''LAC'': Latin America and the Caribbean + + Third level: countries from ISO 3166-1 Country codes. + + Valid regions: * ''GLOBAL'' * ''NA'' * ''JAPAC'' * ''EMEA'' * ''LAC'' + * ''AE'' + + * ''AR'' * ''AT'' * ''AU'' * ''BE'' * ''BR'' * ''CA'' * ''CH'' * ''CL'' + * ''CN'' * ''CO'' + + * ''CZ'' * ''DE'' * ''DK'' * ''DZ'' * ''EC'' * ''EE'' * ''EG'' * ''ES'' + * ''FI'' * ''FR'' + + * ''GB'' * ''GR'' * ''HK'' * ''HU'' * ''ID'' * ''IE'' * ''IL'' * ''IN'' + * ''IR'' * ''IT'' + + * ''JP'' * ''KR'' * ''LV'' * ''MA'' * ''MX'' * ''MY'' * ''NG'' * ''NL'' + * ''NO'' * ''NZ'' + + * ''PE'' * ''PH'' * ''PK'' * ''PL'' * ''PT'' * ''RO'' * ''RS'' * ''RU'' + * ''SA'' * ''SE'' + + * ''SG'' * ''SI'' * ''SK'' * ''TH'' * ''TR'' * ''TW'' * ''UA'' * ''US'' + * ''VE'' * ''VN'' + + * ''ZA''' + isOptional: true + parameterType: LIST + forecasting_predefined_window_column: + defaultValue: '' + description: Forecasting predefined window column. + isOptional: true + parameterType: STRING + forecasting_time_column: + defaultValue: '' + description: Forecasting time column. + isOptional: true + parameterType: STRING + forecasting_time_series_attribute_columns: + defaultValue: [] + description: 'Forecasting + + time series attribute columns.' + isOptional: true + parameterType: LIST + forecasting_time_series_identifier_column: + description: '[Deprecated] A forecasting time series identifier column. + Raises an + + exception if used - use the "time_series_identifier_column" field + + instead.' + isOptional: true + parameterType: STRING + forecasting_time_series_identifier_columns: + defaultValue: [] + description: The list of forecasting time series identifier columns. + isOptional: true + parameterType: LIST + forecasting_unavailable_at_forecast_columns: + defaultValue: [] + description: 'Forecasting + + unavailable at forecast columns.' + isOptional: true + parameterType: LIST + forecasting_window_max_count: + defaultValue: -1.0 + description: Forecasting window max count. + isOptional: true + parameterType: NUMBER_INTEGER + forecasting_window_stride_length: + defaultValue: -1.0 + description: Forecasting window stride length. + isOptional: true + parameterType: NUMBER_INTEGER + group_columns: + isOptional: true + parameterType: LIST + group_temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + group_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + legacy_transformations_path: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + description: Location for the created GCP services. + parameterType: STRING + materialized_examples_format: + defaultValue: tfrecords_gzip + description: 'The format to use for the + + materialized examples. Should be either ''tfrecords_gzip'' (default) or + + ''parquet''.' + isOptional: true + parameterType: STRING + max_selected_features: + defaultValue: 1000.0 + description: 'Maximum number of features to + + select. If specified, the transform config will be purged by only using + + the selected features that ranked top in the feature ranking, which has + + the ranking value for all supported features. If the number of input + + features is smaller than max_selected_features specified, we will still + + run the feature selection process and generate the feature ranking, no + + features will be excluded. The value will be set to 1000 by default if + + run_feature_selection is enabled.' + isOptional: true + parameterType: NUMBER_INTEGER + model_type: + description: 'Model type, which we wish to engineer features + + for. Can be one of: neural_network, boosted_trees, l2l, seq2seq, tft, + or + + tide. Defaults to the empty value, `None`.' + isOptional: true + parameterType: STRING + multimodal_image_columns: + defaultValue: [] + description: 'List of multimodal image + + columns. Defaults to an empty list.' + isOptional: true + parameterType: LIST + multimodal_tabular_columns: + defaultValue: [] + description: 'List of multimodal tabular + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_text_columns: + defaultValue: [] + description: 'List of multimodal text + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + multimodal_timeseries_columns: + defaultValue: [] + description: 'List of multimodal timeseries + + columns. Defaults to an empty list' + isOptional: true + parameterType: LIST + predefined_split_key: + defaultValue: '' + description: Predefined split key. + isOptional: true + parameterType: STRING + prediction_type: + defaultValue: '' + description: 'Model prediction type. One of + + "classification", "regression", "time_series".' + isOptional: true + parameterType: STRING + project: + description: Project to run feature transform engine. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distill: + defaultValue: false + description: '(deprecated) Whether the distillation should be applied + + to the training.' + isOptional: true + parameterType: BOOLEAN + run_feature_selection: + defaultValue: false + description: 'Whether the feature selection + + should be applied to the dataset.' + isOptional: true + parameterType: BOOLEAN + stats_gen_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + statistics generation. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental.' + isOptional: true + parameterType: STRING + stratified_split_key: + defaultValue: '' + description: Stratified split key. + isOptional: true + parameterType: STRING + target_column: + defaultValue: '' + description: Target column of input data. + isOptional: true + parameterType: STRING + temporal_total_weight: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_DOUBLE + test_fraction: + defaultValue: -1.0 + description: Fraction of input data for testing. + isOptional: true + parameterType: NUMBER_DOUBLE + tf_auto_transform_features: + defaultValue: {} + description: "Dict mapping auto and/or type-resolutions to\nTF transform\ + \ features. FTE will automatically configure a set of\nbuilt-in transformations\ + \ for each feature based on its data statistics.\nIf users do not want\ + \ auto type resolution, but want the set of\ntransformations for a given\ + \ type to be automatically generated, they\nmay specify pre-resolved transformations\ + \ types. The following type hint\ndict keys are supported: * 'auto' *\ + \ 'categorical' * 'numeric' * 'text'\n* 'timestamp'\n Example: .. code-block::\ + \ python { \"auto\": [\"feature1\"],\n \"categorical\": [\"feature2\"\ + , \"feature3\"], } Note that the target and\n weight column may not\ + \ be included as an auto transformation unless\n users are running\ + \ forecasting." + isOptional: true + parameterType: STRUCT + tf_custom_transformation_definitions: + defaultValue: [] + description: "List of\nTensorFlow-based custom transformation definitions.\ + \ Custom,\nbring-your-own transform functions, where users can define\ + \ and import\ntheir own transform function and use it with FTE's built-in\n\ + transformations.\n Example: .. code-block:: python [ { \"transformation\"\ + : \"PlusOne\",\n \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"plus_one_transform\" }, { \"transformation\"\ + :\n \"MultiplyTwo\", \"module_path\": \"gs://bucket/custom_transform_fn.py\"\ + ,\n \"function_name\": \"multiply_two_transform\" } ] Using custom\n\ + \ transform function together with FTE's built-in transformations:\ + \ ..\n code-block:: python [ { \"transformation\": \"CastToFloat\"\ + ,\n \"input_columns\": [\"feature_1\"], \"output_columns\": [\"feature_1\"\ + ] },{\n \"transformation\": \"PlusOne\", \"input_columns\": [\"feature_1\"\ + ]\n \"output_columns\": [\"feature_1_plused_one\"] },{ \"transformation\"\ + :\n \"MultiplyTwo\", \"input_columns\": [\"feature_1\"] \"output_columns\"\ + :\n [\"feature_1_multiplied_two\"] } ]" + isOptional: true + parameterType: LIST + tf_transform_execution_engine: + defaultValue: dataflow + description: 'Execution engine to perform + + row-level TF transformations. Can be one of: "dataflow" (by default) or + + "bigquery". Using "bigquery" as the execution engine is experimental and + + is for allowlisted customers only. In addition, executing on "bigquery" + + only supports auto transformations (i.e., specified by + + tf_auto_transform_features) and will raise an error when + + tf_custom_transformation_definitions or tf_transformations_path is set.' + isOptional: true + parameterType: STRING + tf_transformations_path: + defaultValue: '' + description: "Path to TensorFlow-based\ntransformation configuration. Path\ + \ to a JSON file used to specified\nFTE's TF transformation configurations.\ + \ In the following, we provide\nsome sample transform configurations\ + \ to demonstrate FTE's capabilities.\nAll transformations on input columns\ + \ are explicitly specified with FTE's\nbuilt-in transformations. Chaining\ + \ of multiple transformations on a\nsingle column is also supported. For\ + \ example: .. code-block:: python [\n{ \"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }, {\n\"transformation\": \"ZScale\"\ + , \"input_columns\": [\"feature_2\"] } ]\nAdditional information about\ + \ FTE's currently supported built-in\ntransformations:\n Datetime:\ + \ Extracts datetime featues from a column containing\n timestamp\ + \ strings.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"Datetime\", \"input_columns\": [\"feature_1\"], \"time_format\"\ + :\n \"%Y-%m-%d\" }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the datetime\ + \ transformation on.\n output_columns: Names of output\n\ + \ columns, one for each datetime_features element.\n \ + \ time_format: Datetime format string. Time format is\n \ + \ a combination of Date + Time Delimiter (optional) + Time\n\ + \ (optional) directives. Valid date directives are as\n\ + \ follows * '%Y-%m-%d' # 2018-11-30 * '%Y/%m/%d' #\n\ + \ 2018/11/30 * '%y-%m-%d' # 18-11-30 * '%y/%m/%d' #\n\ + \ 18/11/30 * '%m-%d-%Y' # 11-30-2018 * '%m/%d/%Y' #\n\ + \ 11/30/2018 * '%m-%d-%y' # 11-30-18 * '%m/%d/%y' #\n\ + \ 11/30/18 * '%d-%m-%Y' # 30-11-2018 * '%d/%m/%Y' #\n\ + \ 30/11/2018 * '%d-%B-%Y' # 30-November-2018 * '%d-%m-%y'\n\ + \ # 30-11-18 * '%d/%m/%y' # 30/11/18 * '%d-%B-%y' #\n\ + \ 30-November-18 * '%d%m%Y' # 30112018 * '%m%d%Y' \ + \ #\n 11302018 * '%Y%m%d' # 20181130 Valid time delimiters\n\ + \ are as follows * 'T' * ' ' Valid time directives are\ + \ as\n follows * '%H:%M' # 23:59 * '%H:%M:%S'\ + \ #\n 23:59:58 * '%H:%M:%S.%f' # 23:59:58[.123456]\ + \ *\n '%H:%M:%S.%f%z' # 23:59:58[.123456]+0000 *\n \ + \ '%H:%M:%S%z', # 23:59:58+0000\n datetime_features:\ + \ List of datetime\n features to be extract. Each entry\ + \ must be one of *\n 'YEAR' * 'MONTH' * 'DAY' * 'DAY_OF_WEEK'\ + \ * 'DAY_OF_YEAR'\n * 'WEEK_OF_YEAR' * 'QUARTER' * 'HOUR'\ + \ * 'MINUTE' *\n 'SECOND' Defaults to ['YEAR', 'MONTH',\ + \ 'DAY',\n 'DAY_OF_WEEK', 'DAY_OF_YEAR', 'WEEK_OF_YEAR']\n\ + \ Log: Performs the natural log on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Log\",\n \ + \ \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the log transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n ZScale:\ + \ Performs Z-scale normalization on a numeric column.\n Example:\ + \ .. code-block:: python { \"transformation\":\n \"ZScale\"\ + , \"input_columns\": [\"feature_1\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform the z-scale transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n Vocabulary:\ + \ Converts strings to integers, where each unique string\n gets\ + \ a unique integer representation.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"Vocabulary\", \"input_columns\"\ + : [\"feature_1\"] }\n Arguments:\n input_columns:\ + \ A list with a single column to\n perform the vocabulary\ + \ transformation on.\n output_columns: A list with a single\n\ + \ output column name, corresponding to the output of our\n\ + \ transformation.\n top_k: Number of the most\ + \ frequent words\n in the vocabulary to use for generating\ + \ dictionary\n lookup indices. If not specified, all words\ + \ in the\n vocabulary will be used. Defaults to None.\n\ + \ frequency_threshold: Limit the vocabulary\n \ + \ only to words whose number of occurrences in the input\n \ + \ exceeds frequency_threshold. If not specified, all words\n \ + \ in the vocabulary will be included. If both top_k and\n\ + \ frequency_threshold are specified, a word must satisfy\n\ + \ both conditions to be included. Defaults to None.\n \ + \ Categorical: Transforms categorical columns to integer columns.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Categorical\", \"input_columns\": [\"feature_1\"], \"top_k\"\ + : 10 }\n Arguments:\n input_columns: A list with\ + \ a single column to\n perform the categorical transformation\ + \ on.\n output_columns: A list with a single\n \ + \ output column name, corresponding to the output of our\n \ + \ transformation.\n top_k: Number of the most frequent\ + \ words\n in the vocabulary to use for generating dictionary\n\ + \ lookup indices. If not specified, all words in the\n\ + \ vocabulary will be used.\n frequency_threshold:\ + \ Limit the vocabulary\n only to words whose number of\ + \ occurrences in the input\n exceeds frequency_threshold.\ + \ If not specified, all words\n in the vocabulary will\ + \ be included. If both top_k and\n frequency_threshold\ + \ are specified, a word must satisfy\n both conditions\ + \ to be included.\n Reduce: Given a column where each entry is a\ + \ numeric array,\n reduces arrays according to our reduce_mode.\n\ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"Reduce\", \"input_columns\": [\"feature_1\"], \"reduce_mode\"\ + :\n \"MEAN\", \"output_columns\": [\"feature_1_mean\"] }\n\ + \ Arguments:\n input_columns: A list with a single\ + \ column to\n perform the reduce transformation on.\n \ + \ output_columns: A list with a single\n output\ + \ column name, corresponding to the output of our\n transformation.\n\ + \ reduce_mode: One of * 'MAX' * 'MIN' *\n \ + \ 'MEAN' * 'LAST_K' Defaults to 'MEAN'.\n last_k: The number\ + \ of last k elements when\n 'LAST_K' reduce mode is used.\ + \ Defaults to 1.\n SplitString: Given a column of strings, splits\ + \ strings into token\n arrays.\n Example: .. code-block::\ + \ python { \"transformation\":\n \"SplitString\", \"input_columns\"\ + : [\"feature_1\"], \"separator\":\n \"$\" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the split string transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ separator: Separator to split input string\n into tokens.\ + \ Defaults to ' '.\n missing_token: Missing token to use\ + \ when\n no string is included. Defaults to ' _MISSING_\ + \ '.\n NGram: Given a column of strings, splits strings into token\ + \ arrays\n where each token is an integer.\n Example:\ + \ .. code-block:: python { \"transformation\": \"NGram\",\n \ + \ \"input_columns\": [\"feature_1\"], \"min_ngram_size\": 1,\n \ + \ \"max_ngram_size\": 2, \"separator\": \" \" }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_ngram_size: Minimum n-gram size. Must\n be a positive\ + \ number and <= max_ngram_size. Defaults to\n 1.\n \ + \ max_ngram_size: Maximum n-gram size. Must\n \ + \ be a positive number and >= min_ngram_size. Defaults to\n \ + \ 2.\n top_k: Number of the most frequent words\n \ + \ in the vocabulary to use for generating dictionary\n \ + \ lookup indices. If not specified, all words in the\n \ + \ vocabulary will be used. Defaults to None.\n \ + \ frequency_threshold: Limit the\n dictionary's vocabulary\ + \ only to words whose number of\n occurrences in the input\ + \ exceeds frequency_threshold. If\n not specified, all\ + \ words in the vocabulary will be\n included. If both top_k\ + \ and frequency_threshold are\n specified, a word must\ + \ satisfy both conditions to be\n included. Defaults to\ + \ None.\n separator: Separator to split input string\n \ + \ into tokens. Defaults to ' '.\n missing_token:\ + \ Missing token to use when\n no string is included. Defaults\ + \ to ' _MISSING_ '.\n Clip: Given a numeric column, clips elements\ + \ such that elements <\n min_value are assigned min_value, and\ + \ elements > max_value are\n assigned max_value.\n Example:\ + \ .. code-block:: python { \"transformation\": \"Clip\",\n \ + \ \"input_columns\": [\"col1\"], \"output_columns\":\n [\"\ + col1_clipped\"], \"min_value\": 1., \"max_value\": 10., }\n Arguments:\n\ + \ input_columns: A list with a single column to\n \ + \ perform the n-gram transformation on.\n output_columns:\ + \ A list with a single\n output column name, corresponding\ + \ to the output of our\n transformation.\n \ + \ min_value: Number where all values below\n min_value\ + \ are set to min_value. If no min_value is\n provided,\ + \ min clipping will not occur. Defaults to None.\n max_value:\ + \ Number where all values above\n max_value are set to\ + \ max_value If no max_value is\n provided, max clipping\ + \ will not occur. Defaults to None.\n MultiHotEncoding: Performs\ + \ multi-hot encoding on a categorical\n array column.\n \ + \ Example: .. code-block:: python { \"transformation\":\n \ + \ \"MultiHotEncoding\", \"input_columns\": [\"col1\"], } The number\n\ + \ of classes is determened by the largest number included in\n\ + \ the input if it is numeric or the total number of unique\n\ + \ values of the input if it is type str. If the input is has\n\ + \ type str and an element contians separator tokens, the input\n\ + \ will be split at separator indices, and the each element\ + \ of\n the split list will be considered a seperate class.\ + \ For\n example,\n Input: .. code-block:: python\ + \ [ [\"foo bar\"], # Example\n 0 [\"foo\", \"bar\"],\ + \ # Example 1 [\"foo\"], # Example\n 2 [\"bar\"\ + ], # Example 3 ]\n Output (with default separator=\"\ + \ \"): .. code-block:: python [\n [1, 1], # Example\ + \ 0 [1, 1], # Example 1\n [1, 0], # Example\ + \ 2 [0, 1], # Example 3 ]\n Arguments:\n \ + \ input_columns: A list with a single column to\n perform\ + \ the multi-hot-encoding on.\n output_columns: A list with\ + \ a single\n output column name, corresponding to the output\ + \ of our\n transformation.\n top_k: Number\ + \ of the most frequent words\n in the vocabulary to use\ + \ for generating dictionary\n lookup indices. If not specified,\ + \ all words in the\n vocabulary will be used. Defaults\ + \ to None.\n frequency_threshold: Limit the\n \ + \ dictionary's vocabulary only to words whose number of\n \ + \ occurrences in the input exceeds frequency_threshold. If\n \ + \ not specified, all words in the vocabulary will be\n \ + \ included. If both top_k and frequency_threshold are\n \ + \ specified, a word must satisfy both conditions to be\n\ + \ included. Defaults to None.\n separator:\ + \ Separator to split input string\n into tokens. Defaults\ + \ to ' '.\n MaxAbsScale: Performs maximum absolute scaling on a numeric\n\ + \ column.\n Example: .. code-block:: python { \"transformation\"\ + :\n \"MaxAbsScale\", \"input_columns\": [\"col1\"], \"output_columns\"\ + :\n [\"col1_max_abs_scaled\"] }\n Arguments:\n \ + \ input_columns: A list with a single column to\n \ + \ perform max-abs-scale on.\n output_columns: A list\ + \ with a single\n output column name, corresponding to\ + \ the output of our\n transformation.\n Custom: Transformations\ + \ defined in\n tf_custom_transformation_definitions are included\ + \ here in the\n TensorFlow-based transformation configuration.\ + \ For example,\n given the following tf_custom_transformation_definitions:\ + \ ..\n code-block:: python [ { \"transformation\": \"PlusX\"\ + ,\n \"module_path\": \"gs://bucket/custom_transform_fn.py\",\n\ + \ \"function_name\": \"plus_one_transform\" } ] We can include\ + \ the\n following transformation: .. code-block:: python {\n\ + \ \"transformation\": \"PlusX\", \"input_columns\": [\"col1\"],\n\ + \ \"output_columns\": [\"col1_max_abs_scaled\"] \"x\": 5 } Note\ + \ that\n input_columns must still be included in our arguments\ + \ and\n output_columns is optional. All other arguments are those\n\ + \ defined in custom_transform_fn.py, which includes `\"x\"` in\ + \ this\n case. See tf_custom_transformation_definitions above.\n\ + \ legacy_transformations_path (Optional[str]) Deprecated. Prefer\n\ + \ tf_auto_transform_features. Path to a GCS file containing JSON\n\ + \ string for legacy style transformations. Note that\n legacy_transformations_path\ + \ and tf_auto_transform_features\n cannot both be specified." + isOptional: true + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: Timestamp split key. + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + description: Fraction of input data for training. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: Fraction of input data for validation. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: Weight column of input data. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The ranking of features, all features supported in the + + dataset will be included. For "AMI" algorithm, array features won''t be + + available in the ranking as arrays are not supported yet.' + instance_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The materialized dataset. + training_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + bigquery_downsampled_test_split_uri: + description: 'BigQuery URI for the downsampled test + + split to pass to the batch prediction component during batch explain.' + parameterType: STRING + bigquery_test_split_uri: + description: 'BigQuery URI for the test split to pass to the + + batch prediction component during evaluation.' + parameterType: STRING + bigquery_train_split_uri: + description: 'BigQuery URI for the train split to pass to the + + batch prediction component during distillation.' + parameterType: STRING + bigquery_validation_split_uri: + description: 'BigQuery URI for the validation split to + + pass to the batch prediction component during distillation.' + parameterType: STRING + gcp_resources: + description: 'GCP resources created by this component. For more details, + + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + split_example_counts: + description: 'JSON string of data split example counts for train, + + validate, and test splits.' + parameterType: STRING + comp-get-fte-suffix: + executorLabel: exec-get-fte-suffix + inputDefinitions: + parameters: + bigquery_staging_full_dataset_id: + parameterType: STRING + fte_table: + parameterType: STRING + location: + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-get-table-location: + executorLabel: exec-get-table-location + inputDefinitions: + parameters: + default_location: + defaultValue: '' + description: Location to return if no table was given. + isOptional: true + parameterType: STRING + project: + description: The GCP project. + parameterType: STRING + table: + description: The BigQuery table to get a location for. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-model-evaluation-regression: + executorLabel: exec-model-evaluation-regression + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Vertex model used for evaluation. Must be located in the + same + + region as the location argument. It is used to set the default + + configurations for AutoML and custom-trained models.' + isOptional: true + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'BigQuery table + + with prediction or explanation data to be used for this evaluation. For + + prediction results, the table column should be named "predicted_*".' + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'An artifact with its + + URI pointing toward a GCS directory with prediction or explanation files + + to be used for this evaluation. For prediction results, the files should + + be named "prediction.results-*". For explanation results, the files + + should be named "explanation.results-*".' + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'The disk size (in GB) of the machine + + executing the evaluation run.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The machine type executing the + + evaluation run.' + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + description: 'The max number of workers + + executing the evaluation run.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Service account to run the + + Dataflow job. If not set, Dataflow will use the default worker service + + account. For more details, see + + https://cloud.google.com/dataflow/docs/concepts/secURIty-and-permissions#default_worker_service_account' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + description: 'The number of workers executing the + + evaluation run.' + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + description: ' Customer-managed encryption key options. + + If set, resources created by this pipeline will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource is + + created.' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + description: 'Flag to choose Beam runner. Valid options are + + `DirectRunner` and `Dataflow`.' + isOptional: true + parameterType: STRING + ground_truth_bigquery_source: + defaultValue: '' + description: 'Required for custom tabular. + + The BigQuery table URI representing where the ground truth is located. + + Used to provide ground truth for each prediction instance when they are + + not part of the batch prediction jobs prediction instance.' + isOptional: true + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + description: 'Required for custom tabular and non + + tabular data. The file format for the ground truth files. `jsonl`, + + `csv`, and `bigquery` are the allowed formats.' + isOptional: true + parameterType: STRING + ground_truth_gcs_source: + defaultValue: [] + description: 'Required for custom + + tabular and non tabular data. The GCS URIs representing where the ground + + truth is located. Used to provide ground truth for each prediction + + instance when they are not part of the batch prediction jobs prediction + + instance.' + isOptional: true + parameterType: LIST + location: + defaultValue: us-central1 + description: Location for running the evaluation. + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: prediction.value + description: 'The column name of the field + + containing batch prediction scores. Formatted to be able to find nested + + columns, delimited by `.`.' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + description: 'The file format for the batch + + prediction results. `jsonl`, `csv`, and `bigquery` are the allowed + + formats, from Vertex Batch Prediction.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to run evaluation container. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + target_field_name: + description: 'The target field''s name. Formatted to be able to find + + nested columns, delimited by `.`. Prefixed with ''instance.'' on the + + component for Vertex Batch Prediction.' + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: '`google.RegressionMetrics` representing the regression + + evaluation metrics in GCS.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the Dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: An artifact of a model which to upload a new version to. Only + specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: "The unmanaged container model to be uploaded. The Model can\ + \ be passed from an upstream step or imported via a KFP `dsl.importer`.\n\ + :Examples:\n ::\n\n from kfp import dsl\n from google_cloud_pipeline_components.google_cloud_pipeline_components.types\ + \ import artifact_types\n\n importer_spec = dsl.importer(\n artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model',\n\ + \ artifact_class=artifact_types.UnmanagedContainerModel,\n metadata={\n\ + \ 'containerSpec': { 'imageUri':\n 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod'\n\ + \ }\n })" + isOptional: true + parameters: + description: + defaultValue: '' + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + isOptional: true + parameterType: STRING + display_name: + description: 'The display name of the Model. The name + + can be up to 128 characters long and can be consist of any UTF-8 + + characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model)' + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key spec for a Model. If set, this Model and all sub-resources of this + + Model will be secured by this key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + description: 'Metadata describing the Model''s + + input and output for explanation. Both `explanation_metadata` and `explanation_parameters` + must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata)' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters)' + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your model. Label keys and values can be no longer than 64 + + characters (Unicode codepoints), can only contain lowercase letters, + + numeric characters, underscores and dashes. International characters + + are allowed. See https://goo.gl/xmQnxf for more information and + + examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: 'Optional location to upload this Model to. If + + not set, defaults to `us-central1`.' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to upload this Model to. Defaults to the project in + which the PipelineJob is run. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: Artifact tracking the created Model. + parameters: + gcp_resources: + description: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) + which tracks the upload Model's long-running operation. + parameterType: STRING + comp-prophet-trainer: + executorLabel: exec-prophet-trainer + inputDefinitions: + parameters: + data_granularity_unit: + description: 'String representing the units of time for the + + time column.' + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB + + during training.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-1 + description: 'The dataflow machine type used for + + training.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 10.0 + description: 'The max number of Dataflow + + workers used for training.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used.' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + forecast_horizon: + description: 'The number of time periods into the future for + + which forecasts will be created. Future periods start after the latest + + timestamp for each time series.' + parameterType: NUMBER_INTEGER + location: + description: The GCP region for Vertex AI. + parameterType: STRING + max_num_trials: + defaultValue: 6.0 + description: 'Maximum number of tuning trials to perform + + per time series. There are up to 100 possible combinations to explore + + for each time series. Recommended values to try are 3, 6, and 24.' + isOptional: true + parameterType: NUMBER_INTEGER + optimization_objective: + defaultValue: rmse + description: 'Optimization objective for tuning. Supported + + metrics come from Prophet''s performance_metrics function. These are mse, + + rmse, mae, mape, mdape, smape, and coverage.' + isOptional: true + parameterType: STRING + predefined_split_column: + description: 'The predefined_split column name. A string + + that represents a list of comma separated CSV filenames.' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + source_bigquery_uri: + description: 'The BigQuery table path of format + + bq (str)://bq_project.bq_dataset.bq_table' + parameterType: STRING + target_column: + description: 'Name of the column that the model is to predict + + values for.' + parameterType: STRING + time_column: + description: 'Name of the column that identifies time order in the + + time series.' + parameterType: STRING + time_series_identifier_column: + description: 'Name of the column that identifies + + the time series.' + parameterType: STRING + window_column: + description: 'Name of the column that should be used to filter + + input rows. The column should contain either booleans or string + + booleans; if the value of the row is True, generate a sliding window + + from that row.' + parameterType: STRING + outputDefinitions: + artifacts: + evaluated_examples_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: The UnmanagedContainerModel artifact. + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the custom training + + job.' + parameterType: STRING + comp-table-to-uri: + executorLabel: exec-table-to-uri + inputDefinitions: + artifacts: + table: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + use_bq_prefix: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + parameters: + dataset_id: + parameterType: STRING + project_id: + parameterType: STRING + table_id: + parameterType: STRING + uri: + parameterType: STRING + comp-validate-inputs: + executorLabel: exec-validate-inputs + inputDefinitions: + parameters: + bigquery_destination_uri: + isOptional: true + parameterType: STRING + data_granularity_unit: + isOptional: true + parameterType: STRING + data_source_bigquery_table_path: + isOptional: true + parameterType: STRING + data_source_csv_filenames: + isOptional: true + parameterType: STRING + optimization_objective: + isOptional: true + parameterType: STRING + predefined_split_key: + isOptional: true + parameterType: STRING + source_model_uri: + isOptional: true + parameterType: STRING + target_column: + isOptional: true + parameterType: STRING + test_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + isOptional: true + parameterType: STRING + time_series_identifier_column: + isOptional: true + parameterType: STRING + timestamp_split_key: + isOptional: true + parameterType: STRING + training_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + isOptional: true + parameterType: NUMBER_DOUBLE + window_column: + isOptional: true + parameterType: STRING + window_max_count: + isOptional: true + parameterType: NUMBER_INTEGER + window_stride_length: + isOptional: true + parameterType: NUMBER_INTEGER + comp-wrapped-in-list: + executorLabel: exec-wrapped-in-list + inputDefinitions: + parameters: + value: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: LIST +deploymentSpec: + executors: + exec-bigquery-create-dataset: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_create_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_create_dataset(\n project: str,\n location: str,\n\ + \ dataset: str,\n exists_ok: bool = False,\n) -> NamedTuple('Outputs',\ + \ [('project_id', str), ('dataset_id', str)]):\n \"\"\"Creates a BigQuery\ + \ dataset.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import collections\n\n from google.cloud import bigquery\n # pylint:\ + \ enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n ref\ + \ = client.create_dataset(dataset=dataset, exists_ok=exists_ok)\n return\ + \ collections.namedtuple('Outputs', ['project_id', 'dataset_id'])(\n \ + \ ref.project, ref.dataset_id)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-delete-dataset-with-prefix: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - bigquery_delete_dataset_with_prefix + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef bigquery_delete_dataset_with_prefix(\n project: str,\n \ + \ dataset_prefix: str,\n delete_contents: bool = False,\n) -> None:\n\ + \ \"\"\"Deletes all BigQuery datasets matching the given prefix.\"\"\"\n\ + \ # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project)\n for dataset in client.list_datasets(project=project):\n\ + \ if dataset.dataset_id.startswith(dataset_prefix):\n client.delete_dataset(\n\ + \ dataset=dataset.dataset_id,\n delete_contents=delete_contents)\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-bigquery-query-job: + container: + args: + - --type + - BigqueryQueryJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --payload + - '{"Concat": ["{", "\"configuration\": {", "\"query\": ", "{{$.inputs.parameters[''job_configuration_query'']}}", + ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}", "}"]}' + - --job_configuration_query_override + - '{"Concat": ["{", "\"query\": \"", "{{$.inputs.parameters[''query'']}}", + "\"", ", \"query_parameters\": ", "{{$.inputs.parameters[''query_parameters'']}}", + ", \"destination_encryption_configuration\": {", "\"kmsKeyName\": \"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.bigquery.query_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-build-job-configuration-query: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - build_job_configuration_query + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef build_job_configuration_query(\n project_id: str = '',\n \ + \ dataset_id: str = '',\n table_id: str = '',\n write_disposition:\ + \ str = '',\n priority: str = 'INTERACTIVE',\n) -> dict: # pylint: disable=g-bare-generic\n\ + \ \"\"\"Creates a JobConfigurationQuery object.\"\"\"\n config = {\n \ + \ 'priority': priority,\n }\n if all([project_id, dataset_id, table_id]):\n\ + \ config['destinationTable'] = {\n 'projectId': project_id,\n\ + \ 'datasetId': dataset_id,\n 'tableId': table_id,\n }\n\ + \ if write_disposition:\n config['write_disposition'] = write_disposition\n\ + \ return config\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-feature-transform-engine: + container: + args: + - feature_transform_engine + - '{"Concat": ["--project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--location=", "{{$.inputs.parameters[''location'']}}"]}' + - '{"Concat": ["--dataset_level_custom_transformation_definitions=", "{{$.inputs.parameters[''dataset_level_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--dataset_level_transformations=", "{{$.inputs.parameters[''dataset_level_transformations'']}}"]}' + - '{"Concat": ["--forecasting_time_column=", "{{$.inputs.parameters[''forecasting_time_column'']}}"]}' + - '{"IfPresent": {"InputName": "forecasting_time_series_identifier_column", + "Then": {"Concat": ["--forecasting_time_series_identifier_column=", "{{$.inputs.parameters[''forecasting_time_series_identifier_column'']}}"]}}}' + - '{"Concat": ["--forecasting_time_series_identifier_columns=", "{{$.inputs.parameters[''forecasting_time_series_identifier_columns'']}}"]}' + - '{"Concat": ["--forecasting_time_series_attribute_columns=", "{{$.inputs.parameters[''forecasting_time_series_attribute_columns'']}}"]}' + - '{"Concat": ["--forecasting_unavailable_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_unavailable_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_available_at_forecast_columns=", "{{$.inputs.parameters[''forecasting_available_at_forecast_columns'']}}"]}' + - '{"Concat": ["--forecasting_forecast_horizon=", "{{$.inputs.parameters[''forecasting_forecast_horizon'']}}"]}' + - '{"Concat": ["--forecasting_context_window=", "{{$.inputs.parameters[''forecasting_context_window'']}}"]}' + - '{"Concat": ["--forecasting_predefined_window_column=", "{{$.inputs.parameters[''forecasting_predefined_window_column'']}}"]}' + - '{"Concat": ["--forecasting_window_stride_length=", "{{$.inputs.parameters[''forecasting_window_stride_length'']}}"]}' + - '{"Concat": ["--forecasting_window_max_count=", "{{$.inputs.parameters[''forecasting_window_max_count'']}}"]}' + - '{"Concat": ["--forecasting_holiday_regions=", "{{$.inputs.parameters[''forecasting_holiday_regions'']}}"]}' + - '{"Concat": ["--forecasting_apply_windowing=", "{{$.inputs.parameters[''forecasting_apply_windowing'']}}"]}' + - '{"Concat": ["--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}"]}' + - '{"Concat": ["--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}"]}' + - '{"Concat": ["--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}"]}' + - '{"Concat": ["--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}"]}' + - '{"Concat": ["--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}"]}' + - '{"Concat": ["--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}"]}' + - '{"Concat": ["--stats_gen_execution_engine=", "{{$.inputs.parameters[''stats_gen_execution_engine'']}}"]}' + - '{"Concat": ["--tf_transform_execution_engine=", "{{$.inputs.parameters[''tf_transform_execution_engine'']}}"]}' + - '{"IfPresent": {"InputName": "tf_auto_transform_features", "Then": {"Concat": + ["--tf_auto_transform_features=", "{{$.inputs.parameters[''tf_auto_transform_features'']}}"]}}}' + - '{"Concat": ["--tf_custom_transformation_definitions=", "{{$.inputs.parameters[''tf_custom_transformation_definitions'']}}"]}' + - '{"Concat": ["--tf_transformations_path=", "{{$.inputs.parameters[''tf_transformations_path'']}}"]}' + - '{"Concat": ["--legacy_transformations_path=", "{{$.inputs.parameters[''legacy_transformations_path'']}}"]}' + - '{"Concat": ["--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}"]}' + - '{"Concat": ["--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}"]}' + - '{"Concat": ["--bigquery_staging_full_dataset_id=", "{{$.inputs.parameters[''bigquery_staging_full_dataset_id'']}}"]}' + - '{"Concat": ["--target_column=", "{{$.inputs.parameters[''target_column'']}}"]}' + - '{"Concat": ["--weight_column=", "{{$.inputs.parameters[''weight_column'']}}"]}' + - '{"Concat": ["--prediction_type=", "{{$.inputs.parameters[''prediction_type'']}}"]}' + - '{"IfPresent": {"InputName": "model_type", "Then": {"Concat": ["--model_type=", + "{{$.inputs.parameters[''model_type'']}}"]}}}' + - '{"Concat": ["--multimodal_tabular_columns=", "{{$.inputs.parameters[''multimodal_tabular_columns'']}}"]}' + - '{"Concat": ["--multimodal_timeseries_columns=", "{{$.inputs.parameters[''multimodal_timeseries_columns'']}}"]}' + - '{"Concat": ["--multimodal_text_columns=", "{{$.inputs.parameters[''multimodal_text_columns'']}}"]}' + - '{"Concat": ["--multimodal_image_columns=", "{{$.inputs.parameters[''multimodal_image_columns'']}}"]}' + - '{"Concat": ["--run_distill=", "{{$.inputs.parameters[''run_distill'']}}"]}' + - '{"Concat": ["--run_feature_selection=", "{{$.inputs.parameters[''run_feature_selection'']}}"]}' + - '{"Concat": ["--materialized_examples_format=", "{{$.inputs.parameters[''materialized_examples_format'']}}"]}' + - '{"Concat": ["--max_selected_features=", "{{$.inputs.parameters[''max_selected_features'']}}"]}' + - '{"Concat": ["--feature_selection_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir"]}' + - '{"Concat": ["--feature_selection_algorithm=", "{{$.inputs.parameters[''feature_selection_algorithm'']}}"]}' + - '{"Concat": ["--feature_selection_execution_engine=", "{{$.inputs.parameters[''feature_selection_execution_engine'']}}"]}' + - '{"Concat": ["--feature_ranking_path=", "{{$.outputs.artifacts[''feature_ranking''].uri}}"]}' + - '{"Concat": ["--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt"]}' + - '{"Concat": ["--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}"]}' + - '{"Concat": ["--transform_output_artifact_path=", "{{$.outputs.artifacts[''transform_output''].uri}}"]}' + - '{"Concat": ["--transform_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"]}' + - '{"Concat": ["--materialized_examples_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"]}' + - '{"Concat": ["--export_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"]}' + - '{"Concat": ["--materialized_data_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"]}' + - '{"Concat": ["--materialized_data_artifact_path=", "{{$.outputs.artifacts[''materialized_data''].uri}}"]}' + - '{"Concat": ["--bigquery_train_split_uri_path=", "{{$.outputs.parameters[''bigquery_train_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_validation_split_uri_path=", "{{$.outputs.parameters[''bigquery_validation_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--bigquery_downsampled_test_split_uri_path=", "{{$.outputs.parameters[''bigquery_downsampled_test_split_uri''].output_file}}"]}' + - '{"Concat": ["--split_example_counts_path=", "{{$.outputs.parameters[''split_example_counts''].output_file}}"]}' + - '{"Concat": ["--instance_schema_path=", "{{$.outputs.artifacts[''instance_schema''].path}}"]}' + - '{"Concat": ["--training_schema_path=", "{{$.outputs.artifacts[''training_schema''].path}}"]}' + - --job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - '{"Concat": ["--dataflow_project=", "{{$.inputs.parameters[''project'']}}"]}' + - '{"Concat": ["--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging"]}' + - '{"Concat": ["--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp"]}' + - '{"Concat": ["--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}"]}' + - '{"Concat": ["--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}"]}' + - --dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125 + - --feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + - '{"Concat": ["--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}"]}' + - '{"Concat": ["--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}"]}' + - '{"Concat": ["--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}"]}' + - '{"Concat": ["--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}"]}' + - '{"Concat": ["--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + - '{"Concat": ["--autodetect_csv_schema=", "{{$.inputs.parameters[''autodetect_csv_schema'']}}"]}' + - '{"Concat": ["--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}"]}' + - '{"IfPresent": {"InputName": "group_columns", "Then": {"Concat": ["--group_columns=", + "{{$.inputs.parameters[''group_columns'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_total_weight", "Then": {"Concat": ["--group_total_weight=", + "{{$.inputs.parameters[''group_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "temporal_total_weight", "Then": {"Concat": + ["--temporal_total_weight=", "{{$.inputs.parameters[''temporal_total_weight'']}}"]}}}' + - '{"IfPresent": {"InputName": "group_temporal_total_weight", "Then": {"Concat": + ["--group_temporal_total_weight=", "{{$.inputs.parameters[''group_temporal_total_weight'']}}"]}}}' + - '{"Concat": ["--encryption_spec_key_name=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}"]}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125 + exec-get-fte-suffix: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_fte_suffix + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_fte_suffix(\n project: str,\n location: str,\n bigquery_staging_full_dataset_id:\ + \ str,\n fte_table: str,\n) -> str:\n \"\"\"Infers the FTE suffix from\ + \ the intermediate FTE table name.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n client = bigquery.Client(project=project, location=location)\n for\ + \ table in client.list_tables(bigquery_staging_full_dataset_id):\n if\ + \ table.table_id.startswith(fte_table):\n return table.table_id[len(fte_table)\ + \ + 1:]\n raise ValueError(\n f'No FTE output tables found in {bigquery_staging_full_dataset_id}.')\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-get-table-location: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - get_table_location + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef get_table_location(\n project: str,\n table: Optional[str],\n\ + \ default_location: str = '',\n) -> str:\n \"\"\"Returns the region\ + \ the given table belongs to.\n\n Args:\n project: The GCP project.\n\ + \ table: The BigQuery table to get a location for.\n default_location:\ + \ Location to return if no table was given.\n\n Returns:\n A GCP region\ + \ or multi-region.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ from google.cloud import bigquery\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \n if not table:\n return default_location\n\n client = bigquery.Client(project=project)\n\ + \ if table.startswith('bq://'):\n table = table[len('bq://'):]\n elif\ + \ table.startswith('bigquery://'):\n table = table[len('bigquery://'):]\n\ + \ return client.get_table(table).location\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-model-evaluation-regression: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - regression + - --target_field_name + - '{"Concat": ["instance.", "{{$.inputs.parameters[''target_field_name'']}}"]}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - '{"IfPresent": {"InputName": "model", "Then": ["--model_name", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}"]}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --ground_truth_gcs_source + - '{{$.inputs.parameters[''ground_truth_gcs_source'']}}' + - --ground_truth_bigquery_source + - '{{$.inputs.parameters[''ground_truth_bigquery_source'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --dataflow_job_prefix + - evaluation-regression-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"pipeline_job\": + \"", "projects/{{$.inputs.parameters[''project'']}}/locations/{{$.inputs.parameters[''location'']}}/pipelineJobs/{{$.pipeline_job_uuid}}", + "\"", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.model.upload_model.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-prophet-trainer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"prophet-trainer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + ", "\"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, ", "\"job_spec\": {\"worker_pool_specs\": [{\"replica_count\":\"1\", + ", "\"machine_spec\": {\"machine_type\": \"n1-standard-4\"}, ", "\"container_spec\": + {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125\", + ", "\"args\": [\"prophet_trainer\", \"", "--job_name=dataflow-{{$.pipeline_job_name}}\", + \"", "--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125\", + \"", "--prediction_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/fte-prediction-server:20231002_0125\", + \"", "--artifacts_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/model/\", + \"", "--evaluated_examples_dir=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/eval/\", \"", "--region=", "{{$.inputs.parameters[''location'']}}", + "\", \"", "--source_bigquery_uri=", "{{$.inputs.parameters[''source_bigquery_uri'']}}", + "\", \"", "--target_column=", "{{$.inputs.parameters[''target_column'']}}", + "\", \"", "--time_column=", "{{$.inputs.parameters[''time_column'']}}", + "\", \"", "--time_series_identifier_column=", "{{$.inputs.parameters[''time_series_identifier_column'']}}", + "\", \"", "--forecast_horizon=", "{{$.inputs.parameters[''forecast_horizon'']}}", + "\", \"", "--window_column=", "{{$.inputs.parameters[''window_column'']}}", + "\", \"", "--optimization_objective=", "{{$.inputs.parameters[''optimization_objective'']}}", + "\", \"", "--data_granularity_unit=", "{{$.inputs.parameters[''data_granularity_unit'']}}", + "\", \"", "--predefined_split_column=", "{{$.inputs.parameters[''predefined_split_column'']}}", + "\", \"", "--max_num_trials=", "{{$.inputs.parameters[''max_num_trials'']}}", + "\", \"", "--dataflow_project=", "{{$.inputs.parameters[''project'']}}", + "\", \"", "--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"", "--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"", "--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"", "--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"", "--dataflow_subnetwork=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"", "--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"", "--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"", "--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-table-to-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - table_to_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef table_to_uri(\n table: dsl.Input[dsl.Artifact],\n use_bq_prefix:\ + \ bool = False,\n) -> NamedTuple(\n 'Outputs',\n [\n ('project_id',\ + \ str),\n ('dataset_id', str),\n ('table_id', str),\n \ + \ ('uri', str),\n ],\n):\n \"\"\"Converts a google.BQTable to a URI.\"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import collections\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n outputs = [\n table.metadata['projectId'],\n table.metadata['datasetId'],\n\ + \ table.metadata['tableId'],\n ]\n bq_uri = '.'.join(outputs)\n \ + \ if use_bq_prefix:\n bq_uri = 'bq://' + bq_uri\n outputs.append(bq_uri)\n\ + \ return collections.namedtuple(\n 'Outputs',\n ['project_id',\ + \ 'dataset_id', 'table_id', 'uri'],\n )(*outputs)\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-validate-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - validate_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef validate_inputs(\n time_column: Optional[str] = None,\n \ + \ time_series_identifier_column: Optional[str] = None,\n target_column:\ + \ Optional[str] = None,\n data_source_bigquery_table_path: Optional[str]\ + \ = None,\n training_fraction: Optional[float] = None,\n validation_fraction:\ + \ Optional[float] = None,\n test_fraction: Optional[float] = None,\n\ + \ predefined_split_key: Optional[str] = None,\n timestamp_split_key:\ + \ Optional[str] = None,\n data_source_csv_filenames: Optional[str] =\ + \ None,\n source_model_uri: Optional[str] = None,\n bigquery_destination_uri:\ + \ Optional[str] = None,\n window_column: Optional[str] = None,\n window_stride_length:\ + \ Optional[int] = None,\n window_max_count: Optional[int] = None,\n \ + \ optimization_objective: Optional[str] = None,\n data_granularity_unit:\ + \ Optional[str] = None,\n) -> None:\n \"\"\"Checks training pipeline input\ + \ parameters are valid.\"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel\n\ + \ import re\n # pylint: enable=g-import-not-at-top,import-outside-toplevel\n\ + \n project_pattern = r'([a-z0-9.-]+:)?[a-z][a-z0-9-_]{4,28}[a-z0-9]'\n\ + \ dataset_pattern = r'[a-zA-Z0-9_]+'\n table_pattern = r'[^\\.\\:`]+'\n\ + \ dataset_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}')\n\ + \ table_uri_pattern = re.compile(\n f'(bq://)?{project_pattern}[.:]{dataset_pattern}[.:]{table_pattern}')\n\ + \n # Validate BigQuery column and dataset names.\n bigquery_column_parameters\ + \ = [\n time_column,\n time_series_identifier_column,\n target_column,\n\ + \ ]\n column_pattern = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]{1,300}')\n \ + \ for column in bigquery_column_parameters:\n if column and not column_pattern.fullmatch(column):\n\ + \ raise ValueError(f'Invalid column name: {column}.')\n if (bigquery_destination_uri\ + \ and\n not dataset_uri_pattern.fullmatch(bigquery_destination_uri)):\n\ + \ raise ValueError(\n f'Invalid BigQuery dataset URI: {bigquery_destination_uri}.')\n\ + \ if (source_model_uri and not table_uri_pattern.fullmatch(source_model_uri)):\n\ + \ raise ValueError(f'Invalid BigQuery table URI: {source_model_uri}.')\n\ + \n # Validate data source.\n data_source_count = sum([bool(source) for\ + \ source in [\n data_source_bigquery_table_path, data_source_csv_filenames]])\n\ + \ if data_source_count > 1:\n raise ValueError(f'Expected 1 data source,\ + \ found {data_source_count}.')\n if (data_source_bigquery_table_path\n\ + \ and not table_uri_pattern.fullmatch(data_source_bigquery_table_path)):\n\ + \ raise ValueError(\n f'Invalid BigQuery table URI: {data_source_bigquery_table_path}.')\n\ + \ gcs_path_pattern = re.compile(r'gs:\\/\\/(.+)\\/([^\\/]+)')\n if data_source_csv_filenames:\n\ + \ csv_list = [filename.strip()\n for filename in data_source_csv_filenames.split(',')]\n\ + \ for gcs_path in csv_list:\n if not gcs_path_pattern.fullmatch(gcs_path):\n\ + \ raise ValueError(f'Invalid path to CSV stored in GCS: {gcs_path}.')\n\ + \n # Validate split spec.\n fraction_splits = [\n training_fraction,\n\ + \ validation_fraction,\n test_fraction,\n ]\n fraction_splits\ + \ = [None if fraction == -1 else fraction\n for fraction\ + \ in fraction_splits]\n split_count = sum([\n bool(source)\n \ + \ for source in [predefined_split_key,\n any(fraction_splits)]\n\ + \ ])\n if split_count > 1:\n raise ValueError(f'Expected 1 split type,\ + \ found {split_count}.')\n if (predefined_split_key and\n not column_pattern.fullmatch(predefined_split_key)):\n\ + \ raise ValueError(f'Invalid column name: {predefined_split_key}.')\n\ + \ if any(fraction_splits):\n if not all(fraction_splits):\n raise\ + \ ValueError(\n f'All fractions must be non-zero. Got: {fraction_splits}.')\n\ + \ if sum(fraction_splits) != 1:\n raise ValueError(\n f'Fraction\ + \ splits must sum to 1. Got: {sum(fraction_splits)}.')\n if (timestamp_split_key\ + \ and\n not column_pattern.fullmatch(timestamp_split_key)):\n raise\ + \ ValueError(f'Invalid column name: {timestamp_split_key}.')\n if timestamp_split_key\ + \ and not all(fraction_splits):\n raise ValueError('All fractions must\ + \ be non-zero for timestamp split.')\n\n # Validate window config.\n if\ + \ window_stride_length == -1:\n window_stride_length = None\n if window_max_count\ + \ == -1:\n window_max_count = None\n window_configs = [window_column,\ + \ window_stride_length, window_max_count]\n window_config_count = sum([bool(config)\ + \ for config in window_configs])\n if window_config_count > 1:\n raise\ + \ ValueError(f'Expected 1 window config, found {window_config_count}.')\n\ + \ if window_column and not column_pattern.fullmatch(window_column):\n \ + \ raise ValueError(f'Invalid column name: {window_column}.')\n if window_stride_length\ + \ and (window_stride_length < 1 or\n window_stride_length\ + \ > 1000):\n raise ValueError('Stride must be between 1 and 1000. Got:\ + \ '\n f'{window_stride_length}.')\n if window_max_count\ + \ and (window_max_count < 1000 or\n window_max_count\ + \ > int(1e8)):\n raise ValueError('Max count must be between 1000 and\ + \ 100000000. Got: '\n f'{window_max_count}.')\n\n #\ + \ Validate eval metric.\n valid_optimization_objectives = ['rmse', 'mae',\ + \ 'rmsle']\n if optimization_objective:\n if optimization_objective\ + \ not in valid_optimization_objectives:\n raise ValueError(\n \ + \ 'Optimization objective should be one of the following: '\n \ + \ f'{valid_optimization_objectives}, got: {optimization_objective}.')\n\ + \n # Validate data granularity unit.\n valid_data_granularity_units =\ + \ [\n 'minute', 'hour', 'day', 'week', 'month', 'year']\n if data_granularity_unit:\n\ + \ if data_granularity_unit not in valid_data_granularity_units:\n \ + \ raise ValueError(\n 'Granularity unit should be one of the\ + \ following: '\n f'{valid_data_granularity_units}, got: {data_granularity_unit}.')\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-wrapped-in-list: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - wrapped_in_list + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef wrapped_in_list(value: str) -> List[str]:\n \"\"\"Wraps a string\ + \ in a list.\"\"\"\n return [value]\n\n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 +pipelineInfo: + description: Trains one Prophet model per time series. + name: prophet-train +root: + dag: + tasks: + bigquery-delete-dataset-with-prefix: + cachingOptions: {} + componentRef: + name: comp-bigquery-delete-dataset-with-prefix + dependentTasks: + - exit-handler-1 + inputs: + parameters: + dataset_prefix: + runtimeValue: + constant: tmp_{{$.pipeline_job_uuid}} + delete_contents: + runtimeValue: + constant: true + project: + componentInputParameter: project + taskInfo: + name: delete-tmp-dataset + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + inputs: + parameters: + pipelinechannel--data_granularity_unit: + componentInputParameter: data_granularity_unit + pipelinechannel--data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + pipelinechannel--data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--forecast_horizon: + componentInputParameter: forecast_horizon + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--max_num_trials: + componentInputParameter: max_num_trials + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--time_column: + componentInputParameter: time_column + pipelinechannel--time_series_identifier_column: + componentInputParameter: time_series_identifier_column + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--trainer_dataflow_disk_size_gb: + componentInputParameter: trainer_dataflow_disk_size_gb + pipelinechannel--trainer_dataflow_machine_type: + componentInputParameter: trainer_dataflow_machine_type + pipelinechannel--trainer_dataflow_max_num_workers: + componentInputParameter: trainer_dataflow_max_num_workers + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--window_column: + componentInputParameter: window_column + pipelinechannel--window_max_count: + componentInputParameter: window_max_count + pipelinechannel--window_stride_length: + componentInputParameter: window_stride_length + taskInfo: + name: exit-handler-1 + inputDefinitions: + parameters: + data_granularity_unit: + description: 'String representing the units of time for the time + + column.' + parameterType: STRING + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used.' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluation_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB during + + evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-1 + description: 'The dataflow machine type used for + + evaluation.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 10.0 + description: 'The max number of Dataflow workers used + + for evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + forecast_horizon: + description: 'The number of time periods into the future for which + + forecasts will be created. Future periods start after the latest timestamp + + for each time series.' + parameterType: NUMBER_INTEGER + location: + description: The GCP region for Vertex AI. + parameterType: STRING + max_num_trials: + defaultValue: 6.0 + description: 'Maximum number of tuning trials to perform per time series. + + There are up to 100 possible combinations to explore for each time series. + + Recommended values to try are 3, 6, and 24.' + isOptional: true + parameterType: NUMBER_INTEGER + optimization_objective: + description: Optimization objective for the model. + parameterType: STRING + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_evaluation: + defaultValue: true + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + target_column: + description: Name of the column that the model is to predict values for. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + time_column: + description: 'Name of the column that identifies time order in the time + + series.' + parameterType: STRING + time_series_identifier_column: + description: 'Name of the column that identifies the time + + series.' + parameterType: STRING + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + trainer_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB during + + training.' + isOptional: true + parameterType: NUMBER_INTEGER + trainer_dataflow_machine_type: + defaultValue: n1-standard-1 + description: The dataflow machine type used for training. + isOptional: true + parameterType: STRING + trainer_dataflow_max_num_workers: + defaultValue: 10.0 + description: 'The max number of Dataflow workers used + + for training.' + isOptional: true + parameterType: NUMBER_INTEGER + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + window_column: + defaultValue: '' + description: 'Name of the column that should be used to filter input rows. + + The column should contain either booleans or string booleans; if the value + + of the row is True, generate a sliding window from that row.' + isOptional: true + parameterType: STRING + window_max_count: + defaultValue: -1.0 + description: 'Number of rows that should be used to generate input + + examples. If the total row count is larger than this number, the input + + data will be randomly sampled to hit the count.' + isOptional: true + parameterType: NUMBER_INTEGER + window_stride_length: + defaultValue: -1.0 + description: 'Step length used to generate input examples. Every + + window_stride_length rows will be used to generate a sliding window.' + isOptional: true + parameterType: NUMBER_INTEGER +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/utils.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/utils.py new file mode 100644 index 0000000000..b69d5430a5 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/forecasting/utils.py @@ -0,0 +1,341 @@ +"""Util functions for Vertex Forecasting pipelines.""" + +import os +import pathlib +from typing import Any, Dict, Tuple + +_GCPC_FORECASTING_PATH = pathlib.Path(__file__).parent.resolve() + + +def get_bqml_arima_train_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + time_column: str, + time_series_identifier_column: str, + target_column: str, + forecast_horizon: int, + data_granularity_unit: str, + predefined_split_key: str = '', + timestamp_split_key: str = '', + training_fraction: float = -1.0, + validation_fraction: float = -1.0, + test_fraction: float = -1.0, + data_source_csv_filenames: str = '', + data_source_bigquery_table_path: str = '', + window_column: str = '', + window_stride_length: int = -1, + window_max_count: int = -1, + bigquery_destination_uri: str = '', + override_destination: bool = False, + max_order: int = 5, + run_evaluation: bool = True, +) -> Tuple[str, Dict[str, Any]]: + """Get the BQML ARIMA_PLUS training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region for Vertex AI. + root_dir: The Cloud Storage location to store the output. + time_column: Name of the column that identifies time order in the time + series. + time_series_identifier_column: Name of the column that identifies the time + series. + target_column: Name of the column that the model is to predict values for. + forecast_horizon: The number of time periods into the future for which + forecasts will be created. Future periods start after the latest timestamp + for each time series. + data_granularity_unit: The data granularity unit. Accepted values are: + minute, hour, day, week, month, year. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + window_column: Name of the column that should be used to filter input rows. + The column should contain either booleans or string booleans; if the value + of the row is True, generate a sliding window from that row. + window_stride_length: Step length used to generate input examples. Every + window_stride_length rows will be used to generate a sliding window. + window_max_count: Number of rows that should be used to generate input + examples. If the total row count is larger than this number, the input + data will be randomly sampled to hit the count. + bigquery_destination_uri: URI of the desired destination dataset. If not + specified, resources will be created under a new dataset in the project. + Unlike in Vertex Forecasting, all resources will be given hardcoded names + under this dataset, and the model artifact will also be exported here. + override_destination: Whether to overwrite the metrics and evaluated + examples tables if they already exist. If this is False and the tables + exist, this pipeline will fail. + max_order: Integer between 1 and 5 representing the size of the parameter + search space for ARIMA_PLUS. 5 would result in the highest accuracy model, + but also the longest training runtime. + run_evaluation: Whether to run evaluation steps during training. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'time_column': time_column, + 'time_series_identifier_column': time_series_identifier_column, + 'target_column': target_column, + 'forecast_horizon': forecast_horizon, + 'data_granularity_unit': data_granularity_unit, + 'predefined_split_key': predefined_split_key, + 'timestamp_split_key': timestamp_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'window_column': window_column, + 'window_stride_length': window_stride_length, + 'window_max_count': window_max_count, + 'bigquery_destination_uri': bigquery_destination_uri, + 'override_destination': override_destination, + 'max_order': max_order, + 'run_evaluation': run_evaluation, + } + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, 'bqml_arima_train_pipeline.yaml' + ) + return pipeline_definition_path, parameter_values + + +def get_bqml_arima_predict_pipeline_and_parameters( + project: str, + location: str, + model_name: str, + data_source_csv_filenames: str = '', + data_source_bigquery_table_path: str = '', + bigquery_destination_uri: str = '', + generate_explanation: bool = False, +) -> Tuple[str, Dict[str, Any]]: + """Get the BQML ARIMA_PLUS prediction pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region for Vertex AI. + model_name: ARIMA_PLUS BQML model URI. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + bigquery_destination_uri: URI of the desired destination dataset. If not + specified, a resource will be created under a new dataset in the project. + generate_explanation: Generate explanation along with the batch prediction + results. This will cause the batch prediction output to include + explanations. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = { + 'project': project, + 'location': location, + 'model_name': model_name, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_destination_uri': bigquery_destination_uri, + 'generate_explanation': generate_explanation, + } + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, 'bqml_arima_predict_pipeline.yaml' + ) + return pipeline_definition_path, parameter_values + + +def get_prophet_train_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + time_column: str, + time_series_identifier_column: str, + target_column: str, + forecast_horizon: int, + optimization_objective: str, + data_granularity_unit: str, + predefined_split_key: str = '', + timestamp_split_key: str = '', + training_fraction: float = -1.0, + validation_fraction: float = -1.0, + test_fraction: float = -1.0, + data_source_csv_filenames: str = '', + data_source_bigquery_table_path: str = '', + window_column: str = '', + window_stride_length: int = -1, + window_max_count: int = -1, + max_num_trials: int = 6, + trainer_dataflow_machine_type: str = 'n1-standard-1', + trainer_dataflow_max_num_workers: int = 10, + trainer_dataflow_disk_size_gb: int = 40, + evaluation_dataflow_machine_type: str = 'n1-standard-1', + evaluation_dataflow_max_num_workers: int = 10, + evaluation_dataflow_disk_size_gb: int = 40, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + run_evaluation: bool = True, +) -> Tuple[str, Dict[str, Any]]: + """Returns Prophet train pipeline and formatted parameters. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region for Vertex AI. + root_dir: The Cloud Storage location to store the output. + time_column: Name of the column that identifies time order in the time + series. + time_series_identifier_column: Name of the column that identifies the time + series. + target_column: Name of the column that the model is to predict values for. + forecast_horizon: The number of time periods into the future for which + forecasts will be created. Future periods start after the latest timestamp + for each time series. + optimization_objective: Optimization objective for the model. + data_granularity_unit: String representing the units of time for the time + column. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + window_column: Name of the column that should be used to filter input rows. + The column should contain either booleans or string booleans; if the value + of the row is True, generate a sliding window from that row. + window_stride_length: Step length used to generate input examples. Every + window_stride_length rows will be used to generate a sliding window. + window_max_count: Number of rows that should be used to generate input + examples. If the total row count is larger than this number, the input + data will be randomly sampled to hit the count. + max_num_trials: Maximum number of tuning trials to perform per time series. + trainer_dataflow_machine_type: The dataflow machine type used for training. + trainer_dataflow_max_num_workers: The max number of Dataflow workers used + for training. + trainer_dataflow_disk_size_gb: Dataflow worker's disk size in GB during + training. + evaluation_dataflow_machine_type: The dataflow machine type used for + evaluation. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers used + for evaluation. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB during + evaluation. + dataflow_service_account: Custom service account to run dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + run_evaluation: Whether to run evaluation steps during training. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'time_column': time_column, + 'time_series_identifier_column': time_series_identifier_column, + 'target_column': target_column, + 'forecast_horizon': forecast_horizon, + 'predefined_split_key': predefined_split_key, + 'timestamp_split_key': timestamp_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'window_column': window_column, + 'window_stride_length': window_stride_length, + 'window_max_count': window_max_count, + 'max_num_trials': max_num_trials, + 'optimization_objective': optimization_objective, + 'data_granularity_unit': data_granularity_unit, + 'trainer_dataflow_machine_type': trainer_dataflow_machine_type, + 'trainer_dataflow_max_num_workers': trainer_dataflow_max_num_workers, + 'trainer_dataflow_disk_size_gb': trainer_dataflow_disk_size_gb, + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'dataflow_service_account': dataflow_service_account, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'run_evaluation': run_evaluation, + } + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, 'prophet_trainer_pipeline.yaml' + ) + return pipeline_definition_path, parameter_values + + +def get_prophet_prediction_pipeline_and_parameters( + project: str, + location: str, + model_name: str, + time_column: str, + time_series_identifier_column: str, + target_column: str, + data_source_csv_filenames: str = '', + data_source_bigquery_table_path: str = '', + bigquery_destination_uri: str = '', + machine_type: str = 'n1-standard-2', + max_num_workers: int = 10, +) -> Tuple[str, Dict[str, Any]]: + """Returns Prophet prediction pipeline and formatted parameters. + + Unlike the prediction server for Vertex Forecasting, the Prophet prediction + server returns predictions batched by time series id. This pipeline shows how + these predictions can be disaggregated to get results similar to what Vertex + Forecasting provides. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region for Vertex AI. + model_name: The name of the Model resource, in a form of + projects/{project}/locations/{location}/models/{model}. + time_column: Name of the column that identifies time order in the time + series. + time_series_identifier_column: Name of the column that identifies the time + series. + target_column: Name of the column that the model is to predict values for. + data_source_csv_filenames: A string that represents a list of comma + separated CSV filenames. + data_source_bigquery_table_path: The BigQuery table path of format + bq://bq_project.bq_dataset.bq_table + bigquery_destination_uri: URI of the desired destination dataset. If not + specified, resources will be created under a new dataset in the project. + machine_type: The machine type used for batch prediction. + max_num_workers: The max number of workers used for batch prediction. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = { + 'project': project, + 'location': location, + 'model_name': model_name, + 'time_column': time_column, + 'time_series_identifier_column': time_series_identifier_column, + 'target_column': target_column, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'bigquery_destination_uri': bigquery_destination_uri, + 'machine_type': machine_type, + 'max_num_workers': max_num_workers, + } + pipeline_definition_path = os.path.join( + _GCPC_FORECASTING_PATH, 'prophet_predict_pipeline.yaml' + ) + return pipeline_definition_path, parameter_values diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/__init__.py new file mode 100644 index 0000000000..840cd055f5 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/__init__.py @@ -0,0 +1,46 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GA AutoML tabular components.""" + +import os + +from google_cloud_pipeline_components.v1.automl.tabular.cv_trainer import automl_tabular_cv_trainer as CvTrainerOp +from google_cloud_pipeline_components.v1.automl.tabular.ensemble import automl_tabular_ensemble as EnsembleOp +from google_cloud_pipeline_components.v1.automl.tabular.finalizer import automl_tabular_finalizer as FinalizerOp +from google_cloud_pipeline_components.v1.automl.tabular.infra_validator import automl_tabular_infra_validator as InfraValidatorOp +from google_cloud_pipeline_components.v1.automl.tabular.split_materialized_data import split_materialized_data as SplitMaterializedDataOp +from google_cloud_pipeline_components.v1.automl.tabular.stage_1_tuner import automl_tabular_stage_1_tuner as Stage1TunerOp +from google_cloud_pipeline_components.v1.automl.tabular.stats_and_example_gen import tabular_stats_and_example_gen as StatsAndExampleGenOp +from google_cloud_pipeline_components.v1.automl.tabular.training_configurator_and_validator import training_configurator_and_validator as TrainingConfiguratorAndValidatorOp +from google_cloud_pipeline_components.v1.automl.tabular.transform import automl_tabular_transform as TransformOp +from kfp import components + +__all__ = [ + 'CvTrainerOp', + 'InfraValidatorOp', + 'Stage1TunerOp', + 'EnsembleOp', + 'StatsAndExampleGenOp', + 'TransformOp', + 'FinalizerOp', + 'SplitMaterializedDataOp', + 'TrainingConfiguratorAndValidatorOp', +] + +automl_tabular_pipeline = components.load_component_from_file( + # Note, please don't name it as `component.yaml` which will conflict with + # the generated file. + os.path.join(os.path.dirname(__file__), 'automl_tabular_pipeline.yaml') +) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml new file mode 100644 index 0000000000..421cda69d0 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/automl_tabular_pipeline.yaml @@ -0,0 +1,11212 @@ +# PIPELINE DEFINITION +# Name: automl-tabular +# Description: Complete AutoML Tables pipeline. +# Includes feature engineering, architecture search, and hyper-parameter tuning. +# Inputs: +# additional_experiments: dict +# cv_trainer_worker_pool_specs_override: list +# data_source_bigquery_table_path: str [Default: ''] +# data_source_csv_filenames: str [Default: ''] +# dataflow_service_account: str [Default: ''] +# dataflow_subnetwork: str [Default: ''] +# dataflow_use_public_ips: bool [Default: True] +# disable_early_stopping: bool [Default: False] +# distill_batch_predict_machine_type: str [Default: 'n1-standard-16'] +# distill_batch_predict_max_replica_count: int [Default: 25.0] +# distill_batch_predict_starting_replica_count: int [Default: 25.0] +# enable_probabilistic_inference: bool [Default: False] +# encryption_spec_key_name: str [Default: ''] +# evaluation_batch_explain_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_explain_max_replica_count: int [Default: 10.0] +# evaluation_batch_explain_starting_replica_count: int [Default: 10.0] +# evaluation_batch_predict_machine_type: str [Default: 'n1-highmem-8'] +# evaluation_batch_predict_max_replica_count: int [Default: 20.0] +# evaluation_batch_predict_starting_replica_count: int [Default: 20.0] +# evaluation_dataflow_disk_size_gb: int [Default: 50.0] +# evaluation_dataflow_machine_type: str [Default: 'n1-standard-4'] +# evaluation_dataflow_max_num_workers: int [Default: 100.0] +# evaluation_dataflow_starting_num_workers: int [Default: 10.0] +# export_additional_model_without_custom_ops: bool [Default: False] +# fast_testing: bool [Default: False] +# location: str +# model_description: str [Default: ''] +# model_display_name: str [Default: ''] +# optimization_objective: str +# optimization_objective_precision_value: float [Default: -1.0] +# optimization_objective_recall_value: float [Default: -1.0] +# parent_model: system.Artifact +# predefined_split_key: str [Default: ''] +# prediction_type: str +# project: str +# quantiles: list +# root_dir: str +# run_distillation: bool [Default: False] +# run_evaluation: bool [Default: False] +# stage_1_num_parallel_trials: int [Default: 35.0] +# stage_1_tuner_worker_pool_specs_override: list +# stage_1_tuning_result_artifact_uri: str [Default: ''] +# stage_2_num_parallel_trials: int [Default: 35.0] +# stage_2_num_selected_trials: int [Default: 5.0] +# stats_and_example_gen_dataflow_disk_size_gb: int [Default: 40.0] +# stats_and_example_gen_dataflow_machine_type: str [Default: 'n1-standard-16'] +# stats_and_example_gen_dataflow_max_num_workers: int [Default: 25.0] +# stratified_split_key: str [Default: ''] +# study_spec_parameters_override: list +# target_column: str +# test_fraction: float [Default: -1.0] +# timestamp_split_key: str [Default: ''] +# train_budget_milli_node_hours: float +# training_fraction: float [Default: -1.0] +# transform_dataflow_disk_size_gb: int [Default: 40.0] +# transform_dataflow_machine_type: str [Default: 'n1-standard-16'] +# transform_dataflow_max_num_workers: int [Default: 25.0] +# transformations: str +# validation_fraction: float [Default: -1.0] +# vertex_dataset: system.Artifact +# weight_column: str [Default: ''] +# Outputs: +# feature-attribution-2-feature_attributions: system.Metrics +# feature-attribution-3-feature_attributions: system.Metrics +# feature-attribution-feature_attributions: system.Metrics +# model-evaluation-2-evaluation_metrics: system.Metrics +# model-evaluation-3-evaluation_metrics: system.Metrics +# model-evaluation-evaluation_metrics: system.Metrics +components: + comp-automl-tabular-cv-trainer: + executorLabel: exec-automl-tabular-cv-trainer + inputDefinitions: + artifacts: + materialized_cv_splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized cross-validation splits. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: AutoML Tabular tuning result. + parameters: + deadline_hours: + description: Number of hours the cross-validation trainer should run. + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-cv-trainer-2: + executorLabel: exec-automl-tabular-cv-trainer-2 + inputDefinitions: + artifacts: + materialized_cv_splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized cross-validation splits. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: AutoML Tabular tuning result. + parameters: + deadline_hours: + description: Number of hours the cross-validation trainer should run. + parameterType: NUMBER_DOUBLE + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble: + executorLabel: exec-automl-tabular-ensemble + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble-2: + executorLabel: exec-automl-tabular-ensemble-2 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-ensemble-3: + executorLabel: exec-automl-tabular-ensemble-3 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The instance baseline + + used to calculate explanations.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + tuning_result_input: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'AutoML Tabular tuning + + result.' + warmup_data: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: 'The warm up data. Ensemble component will save the + + warm up data together with the model artifact, used to warm up the model + + when prediction server starts.' + isOptional: true + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + export_additional_model_without_custom_ops: + defaultValue: false + description: 'True if export + + an additional model without custom TF operators to the + + `model_without_custom_ops` output.' + isOptional: true + parameterType: BOOLEAN + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model. + model_architecture: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The architecture of the output model. + model_without_custom_ops: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The output model without custom TF operators, this output will + be empty unless `export_additional_model_without_custom_ops` is set. + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + parameters: + explanation_metadata: + description: The explanation parameters used by Vertex online and batch + explanations. + parameterType: STRUCT + explanation_parameters: + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-finalizer: + executorLabel: exec-automl-tabular-finalizer + inputDefinitions: + parameters: + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-infra-validator: + executorLabel: exec-automl-tabular-infra-validator + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-infra-validator-2: + executorLabel: exec-automl-tabular-infra-validator-2 + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-infra-validator-3: + executorLabel: exec-automl-tabular-infra-validator-3 + inputDefinitions: + artifacts: + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'google.UnmanagedContainerModel for model + + to be validated.' + comp-automl-tabular-stage-1-tuner: + executorLabel: exec-automl-tabular-stage-1-tuner + inputDefinitions: + artifacts: + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + disable_early_stopping: + defaultValue: false + description: 'True if disable early stopping. Default + + value is false.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "model_type","categorical_value_spec": {"values": + + ["nn"]}}]' + isOptional: true + parameterType: LIST + tune_feature_selection_rate: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-stage-1-tuner-2: + executorLabel: exec-automl-tabular-stage-1-tuner-2 + inputDefinitions: + artifacts: + feature_ranking: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized eval split. + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'The materialized train + + split.' + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + deadline_hours: + description: 'Number of hours the cross-validation trainer + + should run.' + parameterType: NUMBER_DOUBLE + disable_early_stopping: + defaultValue: false + description: 'True if disable early stopping. Default + + value is false.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + num_parallel_trials: + description: Number of parallel training trials. + parameterType: NUMBER_INTEGER + num_selected_features: + defaultValue: 0.0 + description: 'Number of selected features. The number of + + features to learn in the NN models.' + isOptional: true + parameterType: NUMBER_INTEGER + num_selected_trials: + description: 'Number of selected trials. The number of weak + + learners in the final model is 5 * num_selected_trials.' + parameterType: NUMBER_INTEGER + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + reduce_search_space_mode: + defaultValue: regular + description: 'The reduce search space mode. Possible + + values: "regular" (default), "minimal", "full".' + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + single_run_max_secs: + description: Max number of seconds each training trial runs. + parameterType: NUMBER_INTEGER + study_spec_parameters_override: + defaultValue: [] + description: 'JSON study spec. E.g., + + [{"parameter_id": "model_type","categorical_value_spec": {"values": + + ["nn"]}}]' + isOptional: true + parameterType: LIST + tune_feature_selection_rate: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + worker_pool_specs_override_json: + defaultValue: [] + description: 'JSON worker pool specs. E.g., + + [{"machine_spec": {"machine_type": + + "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": + + "n1-standard-16"}}]' + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + tuning_result_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The trained model and architectures. + parameters: + execution_metrics: + description: Core metrics in dictionary of component execution. + parameterType: STRUCT + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-transform: + executorLabel: exec-automl-tabular-transform + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The eval split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The test split. + train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The train split. + parameters: + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized test split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized train split. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-automl-tabular-transform-2: + executorLabel: exec-automl-tabular-transform-2 + inputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The eval split. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The test split. + train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The train split. + parameters: + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: Location for running the Cross-validation trainer. + parameterType: STRING + project: + description: Project to run Cross-validation trainer. + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + outputDefinitions: + artifacts: + materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized test split. + materialized_test_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The materialized train split. + training_schema_uri: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The training schema. + transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The transform output artifact. + parameters: + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-bool-identity: + executorLabel: exec-bool-identity + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-bool-identity-2: + executorLabel: exec-bool-identity-2 + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-bool-identity-3: + executorLabel: exec-bool-identity-3 + inputDefinitions: + parameters: + value: + description: Boolean value to return + parameterType: BOOLEAN + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-calculate-training-parameters: + executorLabel: exec-calculate-training-parameters + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + run_distillation: + description: Whether to run distill in the training pipeline. + parameterType: BOOLEAN + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + reduce_search_space_mode: + parameterType: STRING + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_num_selected_trials: + parameterType: NUMBER_INTEGER + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-calculate-training-parameters-2: + executorLabel: exec-calculate-training-parameters-2 + inputDefinitions: + parameters: + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + is_skip_architecture_search: + defaultValue: false + description: 'If component is being called in the + + skip_architecture_search pipeline.' + isOptional: true + parameterType: BOOLEAN + run_distillation: + description: Whether to run distill in the training pipeline. + parameterType: BOOLEAN + stage_1_num_parallel_trials: + description: Number of parallel trails for stage 1. + parameterType: NUMBER_INTEGER + stage_2_num_parallel_trials: + description: Number of parallel trails for stage 2. + parameterType: NUMBER_INTEGER + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + outputDefinitions: + parameters: + distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + reduce_search_space_mode: + parameterType: STRING + stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_1_num_selected_trials: + parameterType: NUMBER_INTEGER + stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + stage_2_deadline_hours: + parameterType: NUMBER_DOUBLE + stage_2_single_run_max_secs: + parameterType: NUMBER_INTEGER + comp-condition-2: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-3 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-3 + tasks: + automl-tabular-cv-trainer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-cv-trainer + dependentTasks: + - calculate-training-parameters + - importer + inputs: + artifacts: + materialized_cv_splits: + componentInputArtifact: pipelinechannel--merge-materialized-splits-splits + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + taskInfo: + name: automl-tabular-cv-trainer + automl-tabular-ensemble: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble + dependentTasks: + - automl-tabular-cv-trainer + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-cv-trainer + warmup_data: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble + automl-tabular-infra-validator: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator + dependentTasks: + - automl-tabular-ensemble + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + taskInfo: + name: automl-tabular-infra-validator + bool-identity: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity + calculate-training-parameters: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: true + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters + condition-3: + componentRef: + name: comp-condition-3 + dependentTasks: + - automl-tabular-ensemble + - bool-identity + - model-upload + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble + pipelinechannel--automl-tabular-ensemble-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + pipelinechannel--model-upload-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload + parameters: + pipelinechannel--automl-tabular-ensemble-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble + pipelinechannel--bool-identity-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-Output'] + == 'true' + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: importer + model-upload: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload + dependentTasks: + - automl-tabular-ensemble + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-transform-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--merge-materialized-splits-splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation + tasks: + feature-attribution: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution + dependentTasks: + - model-batch-explanation + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution + model-batch-explanation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation + model-batch-predict: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict + model-evaluation: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation + dependentTasks: + - model-batch-predict + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation + model-evaluation-import: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import + dependentTasks: + - feature-attribution + - model-evaluation + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation + model: + componentInputArtifact: pipelinechannel--model-upload-model + parameters: + dataset_paths: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + dataset_type: + runtimeValue: + constant: tf-record + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-4: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-5 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-7 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-5 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-7 + tasks: + automl-tabular-cv-trainer-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-cv-trainer-2 + dependentTasks: + - automl-tabular-stage-1-tuner + - calculate-training-parameters-2 + inputs: + artifacts: + materialized_cv_splits: + componentInputArtifact: pipelinechannel--merge-materialized-splits-splits + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-stage-1-tuner + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_2_deadline_hours + producerTask: calculate-training-parameters-2 + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_2_single_run_max_secs + producerTask: calculate-training-parameters-2 + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + taskInfo: + name: automl-tabular-cv-trainer-2 + automl-tabular-ensemble-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble-2 + dependentTasks: + - automl-tabular-cv-trainer-2 + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-cv-trainer-2 + warmup_data: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble-2 + automl-tabular-infra-validator-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator-2 + dependentTasks: + - automl-tabular-ensemble-2 + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + taskInfo: + name: automl-tabular-infra-validator-2 + automl-tabular-stage-1-tuner: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-stage-1-tuner + dependentTasks: + - calculate-training-parameters-2 + inputs: + artifacts: + materialized_eval_split: + componentInputArtifact: pipelinechannel--automl-tabular-transform-materialized_eval_split + materialized_train_split: + componentInputArtifact: pipelinechannel--automl-tabular-transform-materialized_train_split + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + componentInputArtifact: pipelinechannel--automl-tabular-transform-transform_output + parameters: + deadline_hours: + taskOutputParameter: + outputParameterKey: stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + taskOutputParameter: + outputParameterKey: stage_1_num_selected_trials + producerTask: calculate-training-parameters-2 + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + taskOutputParameter: + outputParameterKey: reduce_search_space_mode + producerTask: calculate-training-parameters-2 + root_dir: + componentInputParameter: pipelinechannel--root_dir + single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-tabular-stage-1-tuner + bool-identity-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity-2 + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_evaluation + taskInfo: + name: bool-identity-2 + bool-identity-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-bool-identity-3 + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--run_distillation + taskInfo: + name: bool-identity-3 + calculate-training-parameters-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-calculate-training-parameters-2 + inputs: + parameters: + fast_testing: + componentInputParameter: pipelinechannel--fast_testing + is_skip_architecture_search: + runtimeValue: + constant: false + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: calculate-training-parameters-2 + condition-5: + componentRef: + name: comp-condition-5 + dependentTasks: + - automl-tabular-ensemble-2 + - bool-identity-2 + - bool-identity-3 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-2 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-2 + pipelinechannel--bool-identity-2-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-2 + pipelinechannel--bool-identity-3-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-3 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--set-optional-inputs-model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: no-distill + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-3-Output'] + == 'false' + condition-7: + componentRef: + name: comp-condition-7 + dependentTasks: + - automl-tabular-ensemble-2 + - bool-identity-2 + - bool-identity-3 + - calculate-training-parameters-2 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-2 + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + pipelinechannel--tabular-stats-and-example-gen-eval_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + pipelinechannel--tabular-stats-and-example-gen-metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + pipelinechannel--tabular-stats-and-example-gen-test_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-test_split + pipelinechannel--tabular-stats-and-example-gen-train_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-train_split + parameters: + pipelinechannel--bool-identity-2-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-2 + pipelinechannel--bool-identity-3-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: bool-identity-3 + pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours: + taskOutputParameter: + outputParameterKey: distill_stage_1_deadline_hours + producerTask: calculate-training-parameters-2 + pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode: + taskOutputParameter: + outputParameterKey: reduce_search_space_mode + producerTask: calculate-training-parameters-2 + pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs: + taskOutputParameter: + outputParameterKey: stage_1_single_run_max_secs + producerTask: calculate-training-parameters-2 + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + taskInfo: + name: is-distill + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-3-Output'] + == 'true' + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-transform-materialized_eval_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-transform-materialized_train_split: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-transform-transform_output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--merge-materialized-splits-splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-5: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-6 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-6 + tasks: + condition-6: + componentRef: + name: comp-condition-6 + dependentTasks: + - model-upload-2 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + pipelinechannel--model-upload-2-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-2 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + pipelinechannel--bool-identity-2-Output: + componentInputParameter: pipelinechannel--bool-identity-2-Output + pipelinechannel--bool-identity-3-Output: + componentInputParameter: pipelinechannel--bool-identity-3-Output + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-2-Output'] + == 'true' + model-upload-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + parent_model: + componentInputArtifact: pipelinechannel--parent_model + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + description: + componentInputParameter: pipelinechannel--model_description + display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-6: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-2 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation-2 + tasks: + feature-attribution-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-2 + dependentTasks: + - model-batch-explanation-2 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-2 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-2 + model-batch-explanation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-2 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-2-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-2 + model-batch-predict-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-2 + model-evaluation-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-2 + dependentTasks: + - model-batch-predict-2 + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-2 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation-2 + model-evaluation-import-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-2 + dependentTasks: + - feature-attribution-2 + - model-evaluation-2 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-2 + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-2 + model: + componentInputArtifact: pipelinechannel--model-upload-2-model + parameters: + dataset_paths: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + dataset_type: + runtimeValue: + constant: tf-record + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-2-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-2-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-7: + dag: + outputs: + artifacts: + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-8 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-8 + tasks: + automl-tabular-ensemble-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-ensemble-3 + dependentTasks: + - automl-tabular-stage-1-tuner-2 + - automl-tabular-transform-2 + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + instance_baseline: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-instance_baseline + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform-2 + tuning_result_input: + taskOutputArtifact: + outputArtifactKey: tuning_result_output + producerTask: automl-tabular-stage-1-tuner-2 + warmup_data: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-ensemble-3 + automl-tabular-infra-validator-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-infra-validator-3 + dependentTasks: + - automl-tabular-ensemble-3 + inputs: + artifacts: + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + taskInfo: + name: automl-tabular-infra-validator-3 + automl-tabular-stage-1-tuner-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-stage-1-tuner-2 + dependentTasks: + - automl-tabular-transform-2 + inputs: + artifacts: + materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: automl-tabular-transform-2 + materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: automl-tabular-transform-2 + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform-2 + parameters: + deadline_hours: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours + disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + num_selected_trials: + runtimeValue: + constant: 1.0 + project: + componentInputParameter: pipelinechannel--project + reduce_search_space_mode: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_distillation: + runtimeValue: + constant: true + single_run_max_secs: + componentInputParameter: pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs + worker_pool_specs_override_json: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + taskInfo: + name: automl-tabular-stage-1-tuner-2 + automl-tabular-transform-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-transform-2 + dependentTasks: + - write-bp-result-path + - write-bp-result-path-2 + inputs: + artifacts: + dataset_schema: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-dataset_schema + eval_split: + taskOutputArtifact: + outputArtifactKey: result + producerTask: write-bp-result-path-2 + metadata: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-metadata + test_split: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-test_split + train_split: + taskOutputArtifact: + outputArtifactKey: result + producerTask: write-bp-result-path + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-transform-2 + condition-8: + componentRef: + name: comp-condition-8 + dependentTasks: + - automl-tabular-ensemble-3 + - model-upload-3 + inputs: + artifacts: + pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-3 + pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + pipelinechannel--model-upload-3-model: + taskOutputArtifact: + outputArtifactKey: model + producerTask: model-upload-3 + parameters: + pipelinechannel--automl-tabular-ensemble-3-explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-3 + pipelinechannel--bool-identity-2-Output: + componentInputParameter: pipelinechannel--bool-identity-2-Output + pipelinechannel--bool-identity-3-Output: + componentInputParameter: pipelinechannel--bool-identity-3-Output + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--string-not-empty-Output: + componentInputParameter: pipelinechannel--string-not-empty-Output + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + taskInfo: + name: is-evaluation + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--bool-identity-2-Output'] + == 'true' + model-batch-predict-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-3 + dependentTasks: + - read-input-uri + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + taskOutputParameter: + outputParameterKey: Output + producerTask: read-input-uri + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-train-split + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: tf-record + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-3 + model-batch-predict-4: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-4 + dependentTasks: + - read-input-uri-2 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + taskOutputParameter: + outputParameterKey: Output + producerTask: read-input-uri-2 + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-eval-split + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: tf-record + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-4 + model-upload-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-upload-3 + dependentTasks: + - automl-tabular-ensemble-3 + - automl-tabular-infra-validator-3 + inputs: + artifacts: + explanation_metadata_artifact: + taskOutputArtifact: + outputArtifactKey: explanation_metadata_artifact + producerTask: automl-tabular-ensemble-3 + unmanaged_container_model: + taskOutputArtifact: + outputArtifactKey: unmanaged_container_model + producerTask: automl-tabular-ensemble-3 + parameters: + display_name: + runtimeValue: + constant: automl-tabular-distill-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + taskOutputParameter: + outputParameterKey: explanation_parameters + producerTask: automl-tabular-ensemble-3 + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: model-upload-3 + read-input-uri: + cachingOptions: + enableCache: true + componentRef: + name: comp-read-input-uri + inputs: + artifacts: + split_uri: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-train_split + taskInfo: + name: read-input-uri + read-input-uri-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-read-input-uri-2 + inputs: + artifacts: + split_uri: + componentInputArtifact: pipelinechannel--tabular-stats-and-example-gen-eval_split + taskInfo: + name: read-input-uri-2 + write-bp-result-path: + cachingOptions: + enableCache: true + componentRef: + name: comp-write-bp-result-path + dependentTasks: + - model-batch-predict-3 + inputs: + artifacts: + bp_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-3 + taskInfo: + name: write-bp-result-path + write-bp-result-path-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-write-bp-result-path-2 + dependentTasks: + - model-batch-predict-4 + inputs: + artifacts: + bp_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-4 + taskInfo: + name: write-bp-result-path-2 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-2-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + pipelinechannel--tabular-stats-and-example-gen-train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + parameters: + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--calculate-training-parameters-2-distill_stage_1_deadline_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--calculate-training-parameters-2-reduce_search_space_mode: + parameterType: STRING + pipelinechannel--calculate-training-parameters-2-stage_1_single_run_max_secs: + parameterType: NUMBER_INTEGER + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-condition-8: + dag: + outputs: + artifacts: + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature_attributions + producerSubtask: feature-attribution-3 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: evaluation_metrics + producerSubtask: model-evaluation-3 + tasks: + feature-attribution-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-feature-attribution-3 + dependentTasks: + - model-batch-explanation-3 + inputs: + artifacts: + predictions_gcs_source: + taskOutputArtifact: + outputArtifactKey: gcs_output_directory + producerTask: model-batch-explanation-3 + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + taskInfo: + name: feature-attribution-3 + model-batch-explanation-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-explanation-3 + inputs: + artifacts: + explanation_metadata_artifact: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + explanation_parameters: + componentInputParameter: pipelinechannel--automl-tabular-ensemble-3-explanation_parameters + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json + generate_explanation: + runtimeValue: + constant: true + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + taskInfo: + name: model-batch-explanation-3 + model-batch-predict-5: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-batch-predict-5 + inputs: + artifacts: + unmanaged_container_model: + componentInputArtifact: pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model + parameters: + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + gcs_destination_output_uri_prefix: + componentInputParameter: pipelinechannel--root_dir + gcs_source_uris: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + instances_format: + runtimeValue: + constant: tf-record + job_display_name: + runtimeValue: + constant: batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + location: + componentInputParameter: pipelinechannel--location + machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + predictions_format: + runtimeValue: + constant: jsonl + project: + componentInputParameter: pipelinechannel--project + starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + taskInfo: + name: model-batch-predict-5 + model-evaluation-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-3 + dependentTasks: + - model-batch-predict-5 + inputs: + artifacts: + batch_prediction_job: + taskOutputArtifact: + outputArtifactKey: batchpredictionjob + producerTask: model-batch-predict-5 + parameters: + dataflow_disk_size: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + dataflow_max_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + dataflow_workers_num: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + ground_truth_column: + componentInputParameter: pipelinechannel--target_column + ground_truth_format: + runtimeValue: + constant: jsonl + location: + componentInputParameter: pipelinechannel--location + prediction_label_column: + runtimeValue: + constant: '' + prediction_score_column: + runtimeValue: + constant: '' + predictions_format: + runtimeValue: + constant: jsonl + problem_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: model-evaluation-3 + model-evaluation-import-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-model-evaluation-import-3 + dependentTasks: + - feature-attribution-3 + - model-evaluation-3 + inputs: + artifacts: + feature_attributions: + taskOutputArtifact: + outputArtifactKey: feature_attributions + producerTask: feature-attribution-3 + metrics: + taskOutputArtifact: + outputArtifactKey: evaluation_metrics + producerTask: model-evaluation-3 + model: + componentInputArtifact: pipelinechannel--model-upload-3-model + parameters: + dataset_paths: + componentInputParameter: pipelinechannel--tabular-stats-and-example-gen-test_split_json + dataset_type: + runtimeValue: + constant: tf-record + display_name: + runtimeValue: + constant: AutoML Tabular + problem_type: + componentInputParameter: pipelinechannel--prediction_type + taskInfo: + name: model-evaluation-import-3 + inputDefinitions: + artifacts: + pipelinechannel--automl-tabular-ensemble-3-explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--automl-tabular-ensemble-3-unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + pipelinechannel--model-upload-3-model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + pipelinechannel--automl-tabular-ensemble-3-explanation_parameters: + parameterType: STRUCT + pipelinechannel--bool-identity-2-Output: + parameterType: STRING + pipelinechannel--bool-identity-3-Output: + parameterType: STRING + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--location: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--string-not-empty-Output: + parameterType: STRING + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + parameterType: LIST + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-exit-handler-1: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: condition-4 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: condition-4 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: condition-2 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: condition-4 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: condition-4 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: condition-2 + tasks: + automl-tabular-transform: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-transform + dependentTasks: + - tabular-stats-and-example-gen + inputs: + artifacts: + dataset_schema: + taskOutputArtifact: + outputArtifactKey: dataset_schema + producerTask: tabular-stats-and-example-gen + eval_split: + taskOutputArtifact: + outputArtifactKey: eval_split + producerTask: tabular-stats-and-example-gen + metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: tabular-stats-and-example-gen + test_split: + taskOutputArtifact: + outputArtifactKey: test_split + producerTask: tabular-stats-and-example-gen + train_split: + taskOutputArtifact: + outputArtifactKey: train_split + producerTask: tabular-stats-and-example-gen + parameters: + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + project: + componentInputParameter: pipelinechannel--project + root_dir: + componentInputParameter: pipelinechannel--root_dir + taskInfo: + name: automl-tabular-transform + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - automl-tabular-transform + - merge-materialized-splits + - string-not-empty + - tabular-stats-and-example-gen + inputs: + artifacts: + pipelinechannel--automl-tabular-transform-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform + pipelinechannel--merge-materialized-splits-splits: + taskOutputArtifact: + outputArtifactKey: splits + producerTask: merge-materialized-splits + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + taskOutputArtifact: + outputArtifactKey: dataset_schema + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-eval_split: + taskOutputArtifact: + outputArtifactKey: eval_split + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: tabular-stats-and-example-gen + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_distillation: + componentInputParameter: pipelinechannel--run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--set-optional-inputs-model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + taskOutputParameter: + outputParameterKey: downsampled_test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + taskOutputParameter: + outputParameterKey: test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + taskInfo: + name: stage_1_tuning_result_artifact_uri_not_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'true' + condition-4: + componentRef: + name: comp-condition-4 + dependentTasks: + - automl-tabular-transform + - merge-materialized-splits + - string-not-empty + - tabular-stats-and-example-gen + inputs: + artifacts: + pipelinechannel--automl-tabular-transform-materialized_eval_split: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: automl-tabular-transform + pipelinechannel--automl-tabular-transform-materialized_train_split: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: automl-tabular-transform + pipelinechannel--automl-tabular-transform-transform_output: + taskOutputArtifact: + outputArtifactKey: transform_output + producerTask: automl-tabular-transform + pipelinechannel--merge-materialized-splits-splits: + taskOutputArtifact: + outputArtifactKey: splits + producerTask: merge-materialized-splits + pipelinechannel--parent_model: + componentInputArtifact: pipelinechannel--parent_model + pipelinechannel--tabular-stats-and-example-gen-dataset_schema: + taskOutputArtifact: + outputArtifactKey: dataset_schema + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-eval_split: + taskOutputArtifact: + outputArtifactKey: eval_split + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-instance_baseline: + taskOutputArtifact: + outputArtifactKey: instance_baseline + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-metadata: + taskOutputArtifact: + outputArtifactKey: metadata + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-test_split: + taskOutputArtifact: + outputArtifactKey: test_split + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-train_split: + taskOutputArtifact: + outputArtifactKey: train_split + producerTask: tabular-stats-and-example-gen + parameters: + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: pipelinechannel--cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: pipelinechannel--disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: pipelinechannel--distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--distill_batch_predict_starting_replica_count + pipelinechannel--encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: pipelinechannel--evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: pipelinechannel--evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: pipelinechannel--evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: pipelinechannel--evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: pipelinechannel--export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: pipelinechannel--fast_testing + pipelinechannel--location: + componentInputParameter: pipelinechannel--location + pipelinechannel--model_description: + componentInputParameter: pipelinechannel--model_description + pipelinechannel--prediction_type: + componentInputParameter: pipelinechannel--prediction_type + pipelinechannel--project: + componentInputParameter: pipelinechannel--project + pipelinechannel--root_dir: + componentInputParameter: pipelinechannel--root_dir + pipelinechannel--run_distillation: + componentInputParameter: pipelinechannel--run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: pipelinechannel--run_evaluation + pipelinechannel--set-optional-inputs-model_display_name: + componentInputParameter: pipelinechannel--set-optional-inputs-model_display_name + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: pipelinechannel--stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: pipelinechannel--stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: pipelinechannel--stage_2_num_selected_trials + pipelinechannel--string-not-empty-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: string-not-empty + pipelinechannel--study_spec_parameters_override: + componentInputParameter: pipelinechannel--study_spec_parameters_override + pipelinechannel--tabular-stats-and-example-gen-downsampled_test_split_json: + taskOutputParameter: + outputParameterKey: downsampled_test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--tabular-stats-and-example-gen-test_split_json: + taskOutputParameter: + outputParameterKey: test_split_json + producerTask: tabular-stats-and-example-gen + pipelinechannel--target_column: + componentInputParameter: pipelinechannel--target_column + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: pipelinechannel--train_budget_milli_node_hours + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: pipelinechannel--transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: pipelinechannel--transform_dataflow_max_num_workers + taskInfo: + name: stage_1_tuning_result_artifact_uri_empty + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--string-not-empty-Output'] + == 'false' + merge-materialized-splits: + cachingOptions: + enableCache: true + componentRef: + name: comp-merge-materialized-splits + dependentTasks: + - automl-tabular-transform + inputs: + artifacts: + split_0: + taskOutputArtifact: + outputArtifactKey: materialized_train_split + producerTask: automl-tabular-transform + split_1: + taskOutputArtifact: + outputArtifactKey: materialized_eval_split + producerTask: automl-tabular-transform + taskInfo: + name: merge-materialized-splits + string-not-empty: + cachingOptions: + enableCache: true + componentRef: + name: comp-string-not-empty + inputs: + parameters: + value: + componentInputParameter: pipelinechannel--stage_1_tuning_result_artifact_uri + taskInfo: + name: string-not-empty + tabular-stats-and-example-gen: + cachingOptions: + enableCache: true + componentRef: + name: comp-tabular-stats-and-example-gen + inputs: + parameters: + additional_experiments_json: + componentInputParameter: pipelinechannel--additional_experiments + data_source_bigquery_table_path: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: pipelinechannel--set-optional-inputs-data_source_csv_filenames + dataflow_disk_size_gb: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_disk_size_gb + dataflow_machine_type: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_machine_type + dataflow_max_num_workers: + componentInputParameter: pipelinechannel--stats_and_example_gen_dataflow_max_num_workers + dataflow_service_account: + componentInputParameter: pipelinechannel--dataflow_service_account + dataflow_subnetwork: + componentInputParameter: pipelinechannel--dataflow_subnetwork + dataflow_use_public_ips: + componentInputParameter: pipelinechannel--dataflow_use_public_ips + enable_probabilistic_inference: + componentInputParameter: pipelinechannel--enable_probabilistic_inference + encryption_spec_key_name: + componentInputParameter: pipelinechannel--encryption_spec_key_name + location: + componentInputParameter: pipelinechannel--location + optimization_objective: + componentInputParameter: pipelinechannel--optimization_objective + optimization_objective_precision_value: + componentInputParameter: pipelinechannel--optimization_objective_precision_value + optimization_objective_recall_value: + componentInputParameter: pipelinechannel--optimization_objective_recall_value + predefined_split_key: + componentInputParameter: pipelinechannel--predefined_split_key + prediction_type: + componentInputParameter: pipelinechannel--prediction_type + project: + componentInputParameter: pipelinechannel--project + quantiles: + componentInputParameter: pipelinechannel--quantiles + root_dir: + componentInputParameter: pipelinechannel--root_dir + run_distillation: + componentInputParameter: pipelinechannel--run_distillation + stratified_split_key: + componentInputParameter: pipelinechannel--stratified_split_key + target_column_name: + componentInputParameter: pipelinechannel--target_column + test_fraction: + componentInputParameter: pipelinechannel--test_fraction + timestamp_split_key: + componentInputParameter: pipelinechannel--timestamp_split_key + training_fraction: + componentInputParameter: pipelinechannel--training_fraction + transformations: + runtimeValue: + constant: '[]' + transformations_path: + componentInputParameter: pipelinechannel--transformations + validation_fraction: + componentInputParameter: pipelinechannel--validation_fraction + weight_column_name: + componentInputParameter: pipelinechannel--weight_column + taskInfo: + name: tabular-stats-and-example-gen + inputDefinitions: + artifacts: + pipelinechannel--parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--additional_experiments: + parameterType: STRUCT + pipelinechannel--cv_trainer_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--dataflow_service_account: + parameterType: STRING + pipelinechannel--dataflow_subnetwork: + parameterType: STRING + pipelinechannel--dataflow_use_public_ips: + parameterType: BOOLEAN + pipelinechannel--disable_early_stopping: + parameterType: BOOLEAN + pipelinechannel--distill_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--distill_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--distill_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--enable_probabilistic_inference: + parameterType: BOOLEAN + pipelinechannel--encryption_spec_key_name: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_explain_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_explain_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_machine_type: + parameterType: STRING + pipelinechannel--evaluation_batch_predict_max_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_batch_predict_starting_replica_count: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_machine_type: + parameterType: STRING + pipelinechannel--evaluation_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--evaluation_dataflow_starting_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--export_additional_model_without_custom_ops: + parameterType: BOOLEAN + pipelinechannel--fast_testing: + parameterType: BOOLEAN + pipelinechannel--location: + parameterType: STRING + pipelinechannel--model_description: + parameterType: STRING + pipelinechannel--optimization_objective: + parameterType: STRING + pipelinechannel--optimization_objective_precision_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--optimization_objective_recall_value: + parameterType: NUMBER_DOUBLE + pipelinechannel--predefined_split_key: + parameterType: STRING + pipelinechannel--prediction_type: + parameterType: STRING + pipelinechannel--project: + parameterType: STRING + pipelinechannel--quantiles: + parameterType: LIST + pipelinechannel--root_dir: + parameterType: STRING + pipelinechannel--run_distillation: + parameterType: BOOLEAN + pipelinechannel--run_evaluation: + parameterType: BOOLEAN + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + parameterType: STRING + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + parameterType: STRING + pipelinechannel--set-optional-inputs-model_display_name: + parameterType: STRING + pipelinechannel--stage_1_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + parameterType: LIST + pipelinechannel--stage_1_tuning_result_artifact_uri: + parameterType: STRING + pipelinechannel--stage_2_num_parallel_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stage_2_num_selected_trials: + parameterType: NUMBER_INTEGER + pipelinechannel--stats_and_example_gen_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--stats_and_example_gen_dataflow_machine_type: + parameterType: STRING + pipelinechannel--stats_and_example_gen_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--stratified_split_key: + parameterType: STRING + pipelinechannel--study_spec_parameters_override: + parameterType: LIST + pipelinechannel--target_column: + parameterType: STRING + pipelinechannel--test_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--timestamp_split_key: + parameterType: STRING + pipelinechannel--train_budget_milli_node_hours: + parameterType: NUMBER_DOUBLE + pipelinechannel--training_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--transform_dataflow_disk_size_gb: + parameterType: NUMBER_INTEGER + pipelinechannel--transform_dataflow_machine_type: + parameterType: STRING + pipelinechannel--transform_dataflow_max_num_workers: + parameterType: NUMBER_INTEGER + pipelinechannel--transformations: + parameterType: STRING + pipelinechannel--validation_fraction: + parameterType: NUMBER_DOUBLE + pipelinechannel--weight_column: + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + comp-feature-attribution: + executorLabel: exec-feature-attribution + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-2: + executorLabel: exec-feature-attribution-2 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-feature-attribution-3: + executorLabel: exec-feature-attribution-3 + inputDefinitions: + artifacts: + predictions_bigquery_source: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + isOptional: true + predictions_gcs_source: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parameters: + dataflow_disk_size_gb: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 5.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 1.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + force_runner_mode: + defaultValue: '' + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the dataflow + + job. For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-merge-materialized-splits: + executorLabel: exec-merge-materialized-splits + inputDefinitions: + artifacts: + split_0: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The first materialized split. + split_1: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The second materialized split. + outputDefinitions: + artifacts: + splits: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-model-batch-explanation: + executorLabel: exec-model-batch-explanation + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-2: + executorLabel: exec-model-batch-explanation-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-explanation-3: + executorLabel: exec-model-batch-explanation-3 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + instances_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + job_display_name: + parameterType: STRING + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + project: + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-batch-predict: + executorLabel: exec-model-batch-predict + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-2: + executorLabel: exec-model-batch-predict-2 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-3: + executorLabel: exec-model-batch-predict-3 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-4: + executorLabel: exec-model-batch-predict-4 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-batch-predict-5: + executorLabel: exec-model-batch-predict-5 + inputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'The Model used to get predictions via this job. Must share + the same + + ancestor Location. Starting this job has no impact on any existing + + deployments of the Model and their resources. Either this or + + `unmanaged_container_model` must be specified.' + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + description: 'The unmanaged container model used to get predictions via + this job. + + This should be used for models that are not uploaded to Vertex. Either + + this or model must be specified.' + isOptional: true + parameters: + accelerator_count: + defaultValue: 0.0 + description: 'The number of accelerators to attach + + to the `machine_type`. Only used if `machine_type` is set. For more + + details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: NUMBER_INTEGER + accelerator_type: + defaultValue: '' + description: 'The type of accelerator(s) that may be + + attached to the machine as per `accelerator_count`. Only used if + + `machine_type` is set. For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + bigquery_destination_output_uri: + defaultValue: '' + description: 'The BigQuery project location where the output is to be written + to. In + + the given project a new dataset is created with name + + `prediction__` where is made + + BigQuery-dataset-name compatible (for example, most special characters + + become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ + + "based on ISO-8601" format. In the dataset two tables will be created, + + `predictions`, and `errors`. If the Model has both `instance` + + and `prediction` schemata defined then the tables have columns as + + follows: The `predictions` table contains instances for which the + + prediction succeeded, it has columns as per a concatenation of the + + Model''s instance and prediction schemata. The `errors` table + + contains rows for which the prediction has failed, it has instance + + columns, as per the instance schema, followed by a single "errors" + + column, which as values has [google.rpc.Status](Status) + + represented as a STRUCT, and containing only `code` and + + `message`. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + bigquery_source_input_uri: + defaultValue: '' + description: 'BigQuery URI to a table, up to 2000 characters long. For example: + + `projectId.bqDatasetId.bqTableId` For more details about this input + + config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.' + isOptional: true + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + description: 'Customer-managed encryption + + key options for a BatchPredictionJob. If this is set, then all + + resources created by the BatchPredictionJob will be encrypted with the + + provided encryption key. Has the form: + + `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. + + The key needs to be in the same region as where the compute resource + + is created.' + isOptional: true + parameterType: STRING + excluded_fields: + defaultValue: [] + description: 'Fields that will be excluded in the prediction instance that + is + + sent to the Model. + + Excluded will be attached to the batch prediction output if + + key_field is not specified. + + When `excluded_fields` is populated, `included_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord. + + may be specified via the Model''s `parameters_schema_uri`.' + isOptional: true + parameterType: LIST + explanation_metadata: + defaultValue: {} + description: 'Explanation metadata + + configuration for this BatchPredictionJob. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_metadata`. All fields of + + `explanation_metadata` are optional in the request. If a field of the + + `explanation_metadata` object is not populated, the corresponding + + field of the `Model.explanation_metadata` object is inherited. For + + more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata.' + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + description: 'Parameters to configure + + explaining for Model''s predictions. Can be specified only if + + `generate_explanation` is set to `True`. This value overrides the + + value of `Model.explanation_parameters`. All fields of + + `explanation_parameters` are optional in the request. If a field of + + the `explanation_parameters` object is not populated, the + + corresponding field of the `Model.explanation_parameters` object is + + inherited. For more details, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters.' + isOptional: true + parameterType: STRUCT + gcs_destination_output_uri_prefix: + defaultValue: '' + description: 'The Google Cloud + + Storage location of the directory where the output is to be written + + to. In the given directory a new directory is created. Its name is + + `prediction--`, where timestamp + + is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files + + `predictions_0001.`, `predictions_0002.`, + + ..., `predictions_N.` are created where `` + + depends on chosen `predictions_format`, and N may equal 0001 and + + depends on the total number of successfully predicted instances. If + + the Model has both `instance` and `prediction` schemata defined + + then each such file contains predictions as per the + + `predictions_format`. If prediction for any instance failed + + (partially or completely), then an additional + + `errors_0001.`, `errors_0002.`,..., + + `errors_N.` files are created (N depends on total number + + of failed predictions). These files contain the failed instances, as + + per their schema, followed by an additional `error` field which as + + value has `google.rpc.Status` containing only `code` and + + `message` fields. For more details about this output config, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig.' + isOptional: true + parameterType: STRING + gcs_source_uris: + defaultValue: [] + description: 'Google Cloud Storage URI(-s) to your instances to run batch + prediction + + on. They must match `instances_format`. May contain wildcards. For more + + information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). + + For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).' + isOptional: true + parameterType: LIST + generate_explanation: + defaultValue: false + description: 'Generate explanation along with + + the batch prediction results. This will cause the batch prediction + + output to include explanations based on the `prediction_format`: - + + `bigquery`: output includes a column named `explanation`. The value is + + a struct that conforms to the [aiplatform.gapic.Explanation] object. - + + `jsonl`: The JSON objects on each line include an additional entry + + keyed `explanation`. The value of the entry is a JSON object that + + conforms to the [aiplatform.gapic.Explanation] object. - `csv`: + + Generating explanations for CSV format is not supported. If this + + field is set to true, either the Model.explanation_spec or + + explanation_metadata and explanation_parameters must be populated.' + isOptional: true + parameterType: BOOLEAN + included_fields: + defaultValue: [] + description: 'Fields that will be included in the prediction instance that + is + + sent to the Model. + + If `instance_type` is `array`, the order of field names in + + `included_fields` also determines the order of the values in the array. + + When `included_fields` is populated, `excluded_fields` must be empty. + + The input must be JSONL with objects at each line, CSV, BigQuery + + or TfRecord.' + isOptional: true + parameterType: LIST + instance_type: + defaultValue: '' + description: "The format of the instance that the Model\naccepts. Vertex\ + \ AI will convert compatible\n[InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\n\ + to the specified format. Supported values are:\n`object`: Each input is\ + \ converted to JSON object format.\n * For `bigquery`, each row is converted\ + \ to an object.\n * For `jsonl`, each line of the JSONL input must be\ + \ an object.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\ + \ `tf-record-gzip`.\n`array`: Each input is converted to JSON array format.\n\ + \ * For `bigquery`, each row is converted to an array. The order\n \ + \ of columns is determined by the BigQuery column order, unless\n \ + \ [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig)\ + \ is populated.\n `included_fields` must be populated for specifying\ + \ field orders.\n * For `jsonl`, if each line of the JSONL input is an\ + \ object,\n `included_fields` must be populated for specifying field\ + \ orders.\n * Does not apply to `csv`, `file-list`, `tf-record`, or\n\ + \ `tf-record-gzip`.\nIf not specified, Vertex AI converts the batch\ + \ prediction input as\nfollows:\n * For `bigquery` and `csv`, the behavior\ + \ is the same as `array`. The\n order of columns is the same as defined\ + \ in the file or table, unless\n included_fields is populated.\n * For\ + \ `jsonl`, the prediction instance format is determined by\n each line\ + \ of the input.\n * For `tf-record`/`tf-record-gzip`, each record will\ + \ be converted to\n an object in the format of `{\"b64\": }`,\ + \ where `` is\n the Base64-encoded string of the content of the\ + \ record.\n * For `file-list`, each file in the list will be converted\ + \ to an\n object in the format of `{\"b64\": }`, where ``\ + \ is\n the Base64-encoded string of the content of the file." + isOptional: true + parameterType: STRING + instances_format: + defaultValue: jsonl + description: 'The format in which instances are + + given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)''s + supportedInputStorageFormats. + + For more details about this input config, see + + [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.)' + isOptional: true + parameterType: STRING + job_display_name: + description: The user-defined name of this BatchPredictionJob. + parameterType: STRING + key_field: + defaultValue: '' + description: "The name of the field that is considered as a key.\nThe values\ + \ identified by the key field is not included in the\ntransformed instances\ + \ that is sent to the Model. This is similar to\nspecifying this name\ + \ of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig).\ + \ In addition,\nthe batch prediction output will not include the instances.\ + \ Instead the\noutput will only include the value of the key field, in\ + \ a field named\n`key` in the output:\n * For `jsonl` output format, the\ + \ output will have a `key` field\n instead of the `instance` field.\n\ + \ * For `csv`/`bigquery` output format, the output will have have a `key`\n\ + \ column instead of the instance feature columns.\nThe input must be\ + \ JSONL with objects at each line, CSV, BigQuery\nor TfRecord." + isOptional: true + parameterType: STRING + labels: + defaultValue: {} + description: 'The labels with user-defined metadata to + + organize your BatchPredictionJobs. Label keys and values can be no + + longer than 64 characters (Unicode codepoints), can only contain + + lowercase letters, numeric characters, underscores and dashes. + + International characters are allowed. See https://goo.gl/xmQnxf for + + more information and examples of labels.' + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + description: Location for creating the BatchPredictionJob. + isOptional: true + parameterType: STRING + machine_type: + defaultValue: '' + description: 'The type of machine for running batch + + prediction on dedicated resources. If the Model supports + + DEDICATED_RESOURCES this config may be provided (and the job will use + + these resources). If the Model doesn''t support AUTOMATIC_RESOURCES, + + this config must be provided. For more details about the + + BatchDedicatedResources, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. + + For more details about the machine spec, see + + https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec' + isOptional: true + parameterType: STRING + manual_batch_tuning_parameters_batch_size: + defaultValue: 0.0 + description: 'The number of + + the records (e.g. instances) of the operation given in each batch to a + + machine replica. Machine type, and size of a single record should be + + considered when setting this parameter, higher value speeds up the + + batch operation''s execution, but too high value will result in a whole + + batch not fitting in a machine''s memory, and the whole operation will + + fail.' + isOptional: true + parameterType: NUMBER_INTEGER + max_replica_count: + defaultValue: 0.0 + description: 'The maximum number of machine replicas the batch operation + may be scaled + + to. Only used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + model_parameters: + defaultValue: {} + description: The parameters that govern the predictions. The schema of the + parameters + isOptional: true + parameterType: STRUCT + predictions_format: + defaultValue: jsonl + description: 'The format in which Vertex AI gives the predictions. Must + be one of the + + Model''s supportedOutputStorageFormats. + + For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig).' + isOptional: true + parameterType: STRING + project: + defaultValue: '{{$.pipeline_google_cloud_project_id}}' + description: Project to create the BatchPredictionJob. Defaults to the project + in which the PipelineJob is run. + isOptional: true + parameterType: STRING + starting_replica_count: + defaultValue: 0.0 + description: 'The number of machine replicas + + used at the start of the batch operation. If not set, Vertex AI + + decides starting number, not greater than `max_replica_count`. Only + + used if `machine_type` is set.' + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + batchpredictionjob: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + description: '[**Deprecated. Use gcs_output_directory and bigquery_output_table + + instead.**] Artifact + + representation of the created batch prediction job.' + bigquery_output_table: + artifactType: + schemaTitle: google.BQTable + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + bigquery_output_table is specified.' + gcs_output_directory: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: 'Artifact tracking the batch prediction job output. This is + only + + available if + + gcs_destination_output_uri_prefix is specified.' + parameters: + gcp_resources: + description: 'Serialized gcp_resources proto tracking the batch prediction + job. + + For more details, see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + comp-model-evaluation: + executorLabel: exec-model-evaluation + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-2: + executorLabel: exec-model-evaluation-2 + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-3: + executorLabel: exec-model-evaluation-3 + inputDefinitions: + artifacts: + batch_prediction_job: + artifactType: + schemaTitle: google.VertexBatchPredictionJob + schemaVersion: 0.0.1 + parameters: + dataflow_disk_size: + defaultValue: 50.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-4 + isOptional: true + parameterType: STRING + dataflow_max_workers_num: + defaultValue: 100.0 + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + dataflow_workers_num: + defaultValue: 10.0 + isOptional: true + parameterType: NUMBER_INTEGER + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + example_weight_column: + defaultValue: '' + isOptional: true + parameterType: STRING + ground_truth_column: + parameterType: STRING + ground_truth_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + prediction_id_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_label_column: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_score_column: + defaultValue: '' + isOptional: true + parameterType: STRING + predictions_format: + defaultValue: jsonl + isOptional: true + parameterType: STRING + problem_type: + parameterType: STRING + project: + parameterType: STRING + root_dir: + parameterType: STRING + outputDefinitions: + artifacts: + evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-evaluation-import: + executorLabel: exec-model-evaluation-import + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-2: + executorLabel: exec-model-evaluation-import-2 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-evaluation-import-3: + executorLabel: exec-model-evaluation-import-3 + inputDefinitions: + artifacts: + classification_metrics: + artifactType: + schemaTitle: google.ClassificationMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationClassificationOp component.' + isOptional: true + embedding_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The embedding metrics artifact generated from the + + embedding retrieval metrics component.' + isOptional: true + explanation: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'Path for model explanation metrics generated from an evaluation + + component.' + isOptional: true + feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'The feature attributions metrics artifact generated + + from the feature attribution component.' + isOptional: true + forecasting_metrics: + artifactType: + schemaTitle: google.ForecastingMetrics + schemaVersion: 0.0.1 + description: 'google.ForecastingMetrics artifact generated from + + the ModelEvaluationForecastingOp component.' + isOptional: true + metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: Path of metrics generated from an evaluation component. + isOptional: true + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + description: 'Vertex model resource that will be the parent resource of + the + + uploaded evaluation.' + question_answering_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.QuestionAnsweringMetrics.' + isOptional: true + regression_metrics: + artifactType: + schemaTitle: google.RegressionMetrics + schemaVersion: 0.0.1 + description: 'google.ClassificationMetrics artifact generated from + + the ModelEvaluationRegressionOp component.' + isOptional: true + summarization_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.SummarizationMetrics.' + isOptional: true + text_generation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + description: 'system.Metrics artifact generated from + + the LLMEvaluationTextGenerationOp component. Subject to change to + + google.TextGenerationMetrics.' + isOptional: true + parameters: + dataset_path: + defaultValue: '' + isOptional: true + parameterType: STRING + dataset_paths: + defaultValue: [] + isOptional: true + parameterType: LIST + dataset_type: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + defaultValue: '' + description: The display name for the uploaded model evaluation resource. + isOptional: true + parameterType: STRING + problem_type: + description: 'The problem type of the metrics being imported to the + + VertexModel. `classification`, `regression`, `forecasting`, + + `text-generation`, `question-answering`, and `summarization` are the + + currently supported problem types. Must be provided when `metrics` is + + provided.' + isOptional: true + parameterType: STRING + outputDefinitions: + parameters: + evaluation_resource_name: + parameterType: STRING + gcp_resources: + parameterType: STRING + comp-model-upload: + executorLabel: exec-model-upload + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-2: + executorLabel: exec-model-upload-2 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-model-upload-3: + executorLabel: exec-model-upload-3 + inputDefinitions: + artifacts: + explanation_metadata_artifact: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + isOptional: true + parent_model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + isOptional: true + unmanaged_container_model: + artifactType: + schemaTitle: google.UnmanagedContainerModel + schemaVersion: 0.0.1 + isOptional: true + parameters: + description: + defaultValue: '' + isOptional: true + parameterType: STRING + display_name: + parameterType: STRING + encryption_spec_key_name: + defaultValue: '' + isOptional: true + parameterType: STRING + explanation_metadata: + defaultValue: {} + isOptional: true + parameterType: STRUCT + explanation_parameters: + defaultValue: {} + isOptional: true + parameterType: STRUCT + labels: + defaultValue: {} + isOptional: true + parameterType: STRUCT + location: + defaultValue: us-central1 + isOptional: true + parameterType: STRING + project: + parameterType: STRING + outputDefinitions: + artifacts: + model: + artifactType: + schemaTitle: google.VertexModel + schemaVersion: 0.0.1 + parameters: + gcp_resources: + parameterType: STRING + comp-read-input-uri: + executorLabel: exec-read-input-uri + inputDefinitions: + artifacts: + split_uri: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: Tbe path to the file that contains Dataset data. + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-read-input-uri-2: + executorLabel: exec-read-input-uri-2 + inputDefinitions: + artifacts: + split_uri: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: Tbe path to the file that contains Dataset data. + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-set-optional-inputs: + executorLabel: exec-set-optional-inputs + inputDefinitions: + artifacts: + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset when data source is Vertex dataset. + parameters: + data_source_bigquery_table_path: + description: The BigQuery table when data source is BQ. + parameterType: STRING + data_source_csv_filenames: + description: The CSV GCS path when data source is CSV. + parameterType: STRING + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_display_name: + description: The uploaded model's display name. + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + outputDefinitions: + parameters: + data_source_bigquery_table_path: + parameterType: STRING + data_source_csv_filenames: + parameterType: STRING + model_display_name: + parameterType: STRING + comp-string-not-empty: + executorLabel: exec-string-not-empty + inputDefinitions: + parameters: + value: + description: String value to be checked. + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-tabular-stats-and-example-gen: + executorLabel: exec-tabular-stats-and-example-gen + inputDefinitions: + parameters: + additional_experiments: + defaultValue: '' + isOptional: true + parameterType: STRING + additional_experiments_json: + defaultValue: {} + isOptional: true + parameterType: STRUCT + data_source_bigquery_table_path: + defaultValue: '' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + isOptional: true + parameterType: STRING + dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'The disk size, in gigabytes, to use + + on each Dataflow worker instance. If not set, default to 40.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The machine type used for dataflow + + jobs. If not set, default to n1-standard-16.' + isOptional: true + parameterType: STRING + dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The number of workers to run the + + dataflow job. If not set, default to 25.' + isOptional: true + parameterType: NUMBER_INTEGER + dataflow_service_account: + defaultValue: '' + description: 'Custom service account to run + + dataflow jobs.' + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork + + name, when empty the default subnetwork will be used. More + + details: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow + + workers use public IP addresses.' + isOptional: true + parameterType: BOOLEAN + enable_probabilistic_inference: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: Customer-managed encryption key. + isOptional: true + parameterType: STRING + location: + description: 'Location for running dataset statistics and example + + generation.' + parameterType: STRING + optimization_objective: + defaultValue: '' + description: "Objective function the model is optimizing\ntowards. The training\ + \ process creates a model that maximizes/minimizes\nthe value of the objective\ + \ function over the validation set. The\nsupported optimization objectives\ + \ depend on the prediction type. If the\nfield is not set, a default objective\ + \ function is used.\n classification: \"maximize-au-roc\" (default) -\ + \ Maximize the\n area under the receiver operating characteristic (ROC)\ + \ curve.\n \"minimize-log-loss\" - Minimize log loss. \"maximize-au-prc\"\ + \ -\n Maximize the area under the precision-recall curve.\n \"maximize-precision-at-recall\"\ + \ - Maximize precision for a specified\n recall value. \"maximize-recall-at-precision\"\ + \ - Maximize recall for a\n specified precision value.\n classification\ + \ (multi-class): \"minimize-log-loss\" (default) - Minimize\n log loss.\n\ + \ regression: \"minimize-rmse\" (default) - Minimize root-mean-squared\n\ + \ error (RMSE). \"minimize-mae\" - Minimize mean-absolute error (MAE).\n\ + \ \"minimize-rmsle\" - Minimize root-mean-squared log error (RMSLE)." + isOptional: true + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-recall-at-precision". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when + + optimization_objective is "maximize-precision-at-recall". Must be + + between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + predefined_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + prediction_type: + description: 'The prediction type. Supported values: + + "classification", "regression".' + parameterType: STRING + project: + description: 'Project to run dataset statistics and example + + generation.' + parameterType: STRING + quantiles: + defaultValue: [] + isOptional: true + parameterType: LIST + request_type: + defaultValue: COLUMN_STATS_ONLY + isOptional: true + parameterType: STRING + root_dir: + description: The Cloud Storage location to store the output. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'True if in distillation mode. The default value + + is false.' + isOptional: true + parameterType: BOOLEAN + stratified_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + target_column_name: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + timestamp_split_key: + defaultValue: '' + isOptional: true + parameterType: STRING + training_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + transformations: + description: 'Quote escaped JSON string for transformations. Each + + transformation will apply transform function to given input column. And + + the result will be used for training. When creating transformation for + + BigQuery Struct column, the column should be flattened using "." as the + + delimiter.' + parameterType: STRING + transformations_path: + defaultValue: '' + description: 'Path to a GCS file containing JSON + + string for transformations.' + isOptional: true + parameterType: STRING + validation_fraction: + defaultValue: -1.0 + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column_name: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + dataset_schema: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The schema of the dataset. + dataset_stats: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The stats of the dataset. + eval_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The eval split. + instance_baseline: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The instance baseline used to calculate explanations. + metadata: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The tabular example gen metadata. + test_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The test split. + train_split: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + description: The train split. + parameters: + downsampled_test_split_json: + description: The downsampled test split JSON object. + parameterType: LIST + gcp_resources: + description: 'GCP resources created by this component. For more details, + see + + https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.' + parameterType: STRING + test_split_json: + description: The test split JSON object. + parameterType: LIST + comp-write-bp-result-path: + executorLabel: exec-write-bp-result-path + inputDefinitions: + artifacts: + bp_job: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The batch prediction job artifact. + outputDefinitions: + artifacts: + result: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + comp-write-bp-result-path-2: + executorLabel: exec-write-bp-result-path-2 + inputDefinitions: + artifacts: + bp_job: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The batch prediction job artifact. + outputDefinitions: + artifacts: + result: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-automl-tabular-cv-trainer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-cv-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--valid_trials_completed_threshold=0.7\", \"--num_selected_trials=", + "{{$.inputs.parameters[''num_selected_trials'']}}", "\", \"--num_selected_features=", + "{{$.inputs.parameters[''num_selected_features'']}}", "\", \"--lro_job_info=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_cv_splits=", + "{{$.inputs.artifacts[''materialized_cv_splits''].uri}}", "\", \"--tuning_result_input_path=", + "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_custom_job=true\", + \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-cv-trainer-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-cv-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_cv_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--valid_trials_completed_threshold=0.7\", \"--num_selected_trials=", + "{{$.inputs.parameters[''num_selected_trials'']}}", "\", \"--num_selected_features=", + "{{$.inputs.parameters[''num_selected_features'']}}", "\", \"--lro_job_info=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_cv_splits=", + "{{$.inputs.artifacts[''materialized_cv_splits''].uri}}", "\", \"--tuning_result_input_path=", + "{{$.inputs.artifacts[''tuning_result_input''].uri}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_custom_job=true\", + \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-ensemble-3: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"ensemble\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", + \"--custom_model_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--export_custom_model=", "{{$.inputs.parameters[''export_additional_model_without_custom_ops'']}}", + "\", \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", + \"--dataset_schema_path=", "{{$.inputs.artifacts[''dataset_schema''].uri}}", + "\", \"--tuning_result_input_path=", "{{$.inputs.artifacts[''tuning_result_input''].uri}}", + "\", \"--instance_baseline_path=", "{{$.inputs.artifacts[''instance_baseline''].uri}}", + "\", \"--warmup_data=", "{{$.inputs.artifacts[''warmup_data''].uri}}", "\", + \"--prediction_docker_uri=", "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125", + "\", \"--model_path=", "{{$.outputs.artifacts[''model''].uri}}", "\", \"--custom_model_path=", + "{{$.outputs.artifacts[''model_without_custom_ops''].uri}}", "\", \"--explanation_metadata_path=", + "{{$.outputs.parameters[''explanation_metadata''].output_file}}", ",", "{{$.outputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\", \"--explanation_parameters_path=", "{{$.outputs.parameters[''explanation_parameters''].output_file}}", + "\", \"--model_architecture_path=", "{{$.outputs.artifacts[''model_architecture''].uri}}", + "\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-finalizer: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"cancel_l2l_tuner\", \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-infra-validator: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-infra-validator-2: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-infra-validator-3: + container: + args: + - --executor_input + - '{{$}}' + image: us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125 + resources: + cpuLimit: 8.0 + memoryLimit: 52.0 + exec-automl-tabular-stage-1-tuner: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", + "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", + "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--num_selected_features=", "{{$.inputs.parameters[''num_selected_features'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=", + "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=", + "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=", + "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\", + \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-stage-1-tuner-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path=", "{{$.inputs.artifacts[''transform_output''].uri}}", + "\", \"--training_docker_uri=", "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", + "\", \"--feature_selection_result_path=", "{{$.inputs.artifacts[''feature_ranking''].uri}}", + "\", \"--disable_early_stopping=", "{{$.inputs.parameters[''disable_early_stopping'']}}", + "\", \"--tune_feature_selection_rate=", "{{$.inputs.parameters[''tune_feature_selection_rate'']}}", + "\", \"--reduce_search_space_mode=", "{{$.inputs.parameters[''reduce_search_space_mode'']}}", + "\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", + \"--num_parallel_trial=", "{{$.inputs.parameters[''num_parallel_trials'']}}", + "\", \"--single_run_max_secs=", "{{$.inputs.parameters[''single_run_max_secs'']}}", + "\", \"--deadline_hours=", "{{$.inputs.parameters[''deadline_hours'']}}", + "\", \"--num_selected_trials=", "{{$.inputs.parameters[''num_selected_trials'']}}", + "\", \"--num_selected_features=", "{{$.inputs.parameters[''num_selected_features'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--materialized_train_split=", + "{{$.inputs.artifacts[''materialized_train_split''].uri}}", "\", \"--materialized_eval_split=", + "{{$.inputs.artifacts[''materialized_eval_split''].uri}}", "\", \"--is_distill=", + "{{$.inputs.parameters[''run_distillation'']}}", "\", \"--tuning_result_output_path=", + "{{$.outputs.artifacts[''tuning_result_output''].uri}}", "\", \"--kms_key_name=", + "{{$.inputs.parameters[''encryption_spec_key_name'']}}", "\", \"--gcp_resources_path=", + "{{$.outputs.parameters[''gcp_resources''].output_file}}", "\", \"--execution_metrics_path=", + "{{$.outputs.parameters[''execution_metrics''].output_file}}", "\", \"--use_json=true\", + \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-transform: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", + "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", + \"--materialized_splits_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=", + "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=", + "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}", + "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\", + \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}", + "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", + \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", + \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", + "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-automl-tabular-transform-2: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"transform\", \"--is_mp=true\", \"--transform_output_artifact_path=", + "{{$.outputs.artifacts[''transform_output''].uri}}", "\", \"--transform_output_path=", + "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", + \"--materialized_splits_output_path=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\", + \"--metadata_path=", "{{$.inputs.artifacts[''metadata''].uri}}", "\", \"--dataset_schema_path=", + "{{$.inputs.artifacts[''dataset_schema''].uri}}", "\", \"--train_split=", + "{{$.inputs.artifacts[''train_split''].uri}}", "\", \"--eval_split=", "{{$.inputs.artifacts[''eval_split''].uri}}", + "\", \"--test_split=", "{{$.inputs.artifacts[''test_split''].uri}}", "\", + \"--materialized_train_split=", "{{$.outputs.artifacts[''materialized_train_split''].uri}}", + "\", \"--materialized_eval_split=", "{{$.outputs.artifacts[''materialized_eval_split''].uri}}", + "\", \"--materialized_test_split=", "{{$.outputs.artifacts[''materialized_test_split''].uri}}", + "\", \"--training_schema_path=", "{{$.outputs.artifacts[''training_schema_uri''].uri}}", + "\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", + \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", + \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", + "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-bool-identity: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-bool-identity-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-bool-identity-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _bool_identity + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _bool_identity(value: bool) -> str:\n \"\"\"Returns boolean\ + \ value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n\ + \ Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-calculate-training-parameters: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n run_distillation: bool,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_num_selected_trials',\ + \ int),\n ('stage_1_single_run_max_secs', int),\n ('stage_2_deadline_hours',\ + \ float),\n ('stage_2_single_run_max_secs', int),\n ('distill_stage_1_deadline_hours',\ + \ float),\n ('reduce_search_space_mode', str),\n ],\n):\n \"\"\ + \"Calculates training parameters.\n\n Args:\n stage_1_num_parallel_trials:\ + \ Number of parallel trails for stage 1.\n train_budget_milli_node_hours:\ + \ The train budget of creating this model,\n expressed in milli node\ + \ hours i.e. 1,000 value in this field means 1 node\n hour.\n stage_2_num_parallel_trials:\ + \ Number of parallel trails for stage 2.\n run_distillation: Whether\ + \ to run distill in the training pipeline.\n is_skip_architecture_search:\ + \ If component is being called in the\n skip_architecture_search pipeline.\n\ + \ fast_testing: Internal flag used for presubmit tests.\n\n Returns:\n\ + \ stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\ + \ stage_1_num_selected_trials: Number of selected trails for stage\ + \ 1.\n stage_1_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 1\n training trial.\n stage_2_deadline_hours:\ + \ Maximum number of hours to run stage 2.\n stage_2_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 2\n training\ + \ trial.\n distill_stage_1_deadline_hours: Maximum number of hours\ + \ to run stage 1 for\n the model distillation.\n reduce_search_space_mode:\ + \ The reduce search space mode. Possible values:\n minimal, regular,\ + \ full.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ num_folds = 5\n distill_total_trials = 100\n\n stage_1_deadline_hours\ + \ = -1.0\n stage_1_num_selected_trials = -1\n stage_1_single_run_max_secs\ + \ = -1\n stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs =\ + \ -1\n distill_stage_1_deadline_hours = 1.0\n reduce_search_space_mode\ + \ = 'regular'\n\n if is_skip_architecture_search:\n stage_2_deadline_hours\ + \ = train_budget_milli_node_hours / 1000.0\n stage_2_single_run_max_secs\ + \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n else:\n hours = float(train_budget_milli_node_hours)\ + \ / 1000.0\n multiplier = stage_1_num_parallel_trials * hours / 500.0\n\ + \ stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\ + \ phase_2_rounds = int(\n math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\ + \ + 0.5\n )\n if phase_2_rounds < 1:\n phase_2_rounds = 1\n\n\ + \ # All of magic number \"1.3\" above is because the trial doesn't\n\ + \ # always finish in time_per_trial. 1.3 is an empirical safety margin\ + \ here.\n stage_1_deadline_secs = int(\n hours * 3600.0 - 1.3\ + \ * stage_1_single_run_max_secs * phase_2_rounds\n )\n\n if stage_1_deadline_secs\ + \ < hours * 3600.0 * 0.5:\n stage_1_deadline_secs = int(hours * 3600.0\ + \ * 0.5)\n # Phase 1 deadline is the same as phase 2 deadline in this\ + \ case. Phase 2\n # can't finish in time after the deadline is cut,\ + \ so adjust the time per\n # trial to meet the deadline.\n stage_1_single_run_max_secs\ + \ = int(\n stage_1_deadline_secs / (1.3 * phase_2_rounds)\n \ + \ )\n\n reduce_search_space_mode = 'minimal'\n if multiplier > 2:\n\ + \ reduce_search_space_mode = 'regular'\n if multiplier > 4:\n \ + \ reduce_search_space_mode = 'full'\n\n # Stage 2 number of trials\ + \ is stage_1_num_selected_trials *\n # num_folds, which should be equal\ + \ to phase_2_rounds *\n # stage_2_num_parallel_trials. Use this information\ + \ to calculate\n # stage_1_num_selected_trials:\n stage_1_num_selected_trials\ + \ = int(\n phase_2_rounds * stage_2_num_parallel_trials / num_folds\n\ + \ )\n stage_1_deadline_hours = stage_1_deadline_secs / 3600.0\n\n\ + \ stage_2_deadline_hours = hours - stage_1_deadline_hours\n stage_2_single_run_max_secs\ + \ = stage_1_single_run_max_secs\n\n if run_distillation:\n # All\ + \ of magic number \"1.3\" above is because the trial doesn't always\n \ + \ # finish in time_per_trial. 1.3 is an empirical safety margin here.\n\ + \ distill_stage_1_deadline_hours = (\n math.ceil(float(distill_total_trials)\ + \ / stage_1_num_parallel_trials)\n * stage_1_single_run_max_secs\n\ + \ * 1.3\n / 3600.0\n )\n\n if fast_testing:\n \ + \ distill_stage_1_deadline_hours = 0.2\n stage_1_deadline_hours = 0.2\n\ + \ stage_1_single_run_max_secs = 1\n stage_2_deadline_hours = 0.2\n\ + \ stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_num_selected_trials',\n 'stage_1_single_run_max_secs',\n\ + \ 'stage_2_deadline_hours',\n 'stage_2_single_run_max_secs',\n\ + \ 'distill_stage_1_deadline_hours',\n 'reduce_search_space_mode',\n\ + \ ],\n )(\n stage_1_deadline_hours,\n stage_1_num_selected_trials,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ + \ reduce_search_space_mode,\n )\n\n" + image: python:3.7 + exec-calculate-training-parameters-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _calculate_training_parameters + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _calculate_training_parameters(\n stage_1_num_parallel_trials:\ + \ int,\n train_budget_milli_node_hours: float,\n stage_2_num_parallel_trials:\ + \ int,\n run_distillation: bool,\n is_skip_architecture_search: bool\ + \ = False,\n fast_testing: bool = False,\n) -> NamedTuple(\n 'Outputs',\n\ + \ [\n ('stage_1_deadline_hours', float),\n ('stage_1_num_selected_trials',\ + \ int),\n ('stage_1_single_run_max_secs', int),\n ('stage_2_deadline_hours',\ + \ float),\n ('stage_2_single_run_max_secs', int),\n ('distill_stage_1_deadline_hours',\ + \ float),\n ('reduce_search_space_mode', str),\n ],\n):\n \"\"\ + \"Calculates training parameters.\n\n Args:\n stage_1_num_parallel_trials:\ + \ Number of parallel trails for stage 1.\n train_budget_milli_node_hours:\ + \ The train budget of creating this model,\n expressed in milli node\ + \ hours i.e. 1,000 value in this field means 1 node\n hour.\n stage_2_num_parallel_trials:\ + \ Number of parallel trails for stage 2.\n run_distillation: Whether\ + \ to run distill in the training pipeline.\n is_skip_architecture_search:\ + \ If component is being called in the\n skip_architecture_search pipeline.\n\ + \ fast_testing: Internal flag used for presubmit tests.\n\n Returns:\n\ + \ stage_1_deadline_hours: Maximum number of hours to run stage 1.\n\ + \ stage_1_num_selected_trials: Number of selected trails for stage\ + \ 1.\n stage_1_single_run_max_secs: Maximum number seconds to for a\ + \ single stage\n 1\n training trial.\n stage_2_deadline_hours:\ + \ Maximum number of hours to run stage 2.\n stage_2_single_run_max_secs:\ + \ Maximum number seconds to for a single stage\n 2\n training\ + \ trial.\n distill_stage_1_deadline_hours: Maximum number of hours\ + \ to run stage 1 for\n the model distillation.\n reduce_search_space_mode:\ + \ The reduce search space mode. Possible values:\n minimal, regular,\ + \ full.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n import math\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ num_folds = 5\n distill_total_trials = 100\n\n stage_1_deadline_hours\ + \ = -1.0\n stage_1_num_selected_trials = -1\n stage_1_single_run_max_secs\ + \ = -1\n stage_2_deadline_hours = -1.0\n stage_2_single_run_max_secs =\ + \ -1\n distill_stage_1_deadline_hours = 1.0\n reduce_search_space_mode\ + \ = 'regular'\n\n if is_skip_architecture_search:\n stage_2_deadline_hours\ + \ = train_budget_milli_node_hours / 1000.0\n stage_2_single_run_max_secs\ + \ = int(stage_2_deadline_hours * 3600.0 / 1.3)\n else:\n hours = float(train_budget_milli_node_hours)\ + \ / 1000.0\n multiplier = stage_1_num_parallel_trials * hours / 500.0\n\ + \ stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0)\n\ + \ phase_2_rounds = int(\n math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials\ + \ + 0.5\n )\n if phase_2_rounds < 1:\n phase_2_rounds = 1\n\n\ + \ # All of magic number \"1.3\" above is because the trial doesn't\n\ + \ # always finish in time_per_trial. 1.3 is an empirical safety margin\ + \ here.\n stage_1_deadline_secs = int(\n hours * 3600.0 - 1.3\ + \ * stage_1_single_run_max_secs * phase_2_rounds\n )\n\n if stage_1_deadline_secs\ + \ < hours * 3600.0 * 0.5:\n stage_1_deadline_secs = int(hours * 3600.0\ + \ * 0.5)\n # Phase 1 deadline is the same as phase 2 deadline in this\ + \ case. Phase 2\n # can't finish in time after the deadline is cut,\ + \ so adjust the time per\n # trial to meet the deadline.\n stage_1_single_run_max_secs\ + \ = int(\n stage_1_deadline_secs / (1.3 * phase_2_rounds)\n \ + \ )\n\n reduce_search_space_mode = 'minimal'\n if multiplier > 2:\n\ + \ reduce_search_space_mode = 'regular'\n if multiplier > 4:\n \ + \ reduce_search_space_mode = 'full'\n\n # Stage 2 number of trials\ + \ is stage_1_num_selected_trials *\n # num_folds, which should be equal\ + \ to phase_2_rounds *\n # stage_2_num_parallel_trials. Use this information\ + \ to calculate\n # stage_1_num_selected_trials:\n stage_1_num_selected_trials\ + \ = int(\n phase_2_rounds * stage_2_num_parallel_trials / num_folds\n\ + \ )\n stage_1_deadline_hours = stage_1_deadline_secs / 3600.0\n\n\ + \ stage_2_deadline_hours = hours - stage_1_deadline_hours\n stage_2_single_run_max_secs\ + \ = stage_1_single_run_max_secs\n\n if run_distillation:\n # All\ + \ of magic number \"1.3\" above is because the trial doesn't always\n \ + \ # finish in time_per_trial. 1.3 is an empirical safety margin here.\n\ + \ distill_stage_1_deadline_hours = (\n math.ceil(float(distill_total_trials)\ + \ / stage_1_num_parallel_trials)\n * stage_1_single_run_max_secs\n\ + \ * 1.3\n / 3600.0\n )\n\n if fast_testing:\n \ + \ distill_stage_1_deadline_hours = 0.2\n stage_1_deadline_hours = 0.2\n\ + \ stage_1_single_run_max_secs = 1\n stage_2_deadline_hours = 0.2\n\ + \ stage_2_single_run_max_secs = 1\n\n return collections.namedtuple(\n\ + \ 'Outputs',\n [\n 'stage_1_deadline_hours',\n \ + \ 'stage_1_num_selected_trials',\n 'stage_1_single_run_max_secs',\n\ + \ 'stage_2_deadline_hours',\n 'stage_2_single_run_max_secs',\n\ + \ 'distill_stage_1_deadline_hours',\n 'reduce_search_space_mode',\n\ + \ ],\n )(\n stage_1_deadline_hours,\n stage_1_num_selected_trials,\n\ + \ stage_1_single_run_max_secs,\n stage_2_deadline_hours,\n \ + \ stage_2_single_run_max_secs,\n distill_stage_1_deadline_hours,\n\ + \ reduce_search_space_mode,\n )\n\n" + image: python:3.7 + exec-feature-attribution: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-2: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-feature-attribution-3: + container: + args: + - --task + - explanation + - --setup_file + - /setup.py + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --root_dir + - '{{$.pipeline_root}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - '{"IfPresent": {"InputName": "predictions_gcs_source", "Then": ["--batch_prediction_gcs_source", + "{{$.inputs.artifacts[''predictions_gcs_source''].uri}}"]}}' + - '{"IfPresent": {"InputName": "predictions_bigquery_source", "Then": ["--batch_prediction_bigquery_source", + {"Concat": ["bq://", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''projectId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''datasetId'']}}", + ".", "{{$.inputs.artifacts[''predictions_bigquery_source''].metadata[''tableId'']}}"]}]}}' + - --dataflow_job_prefix + - evaluation-feautre-attribution-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size_gb'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --force_runner_mode + - '{{$.inputs.parameters[''force_runner_mode'']}}' + - --gcs_output_path + - '{{$.outputs.artifacts[''feature_attributions''].path}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.9.2 + exec-importer: + importer: + artifactUri: + runtimeParameter: uri + typeSchema: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + exec-merge-materialized-splits: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _merge_materialized_splits + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _merge_materialized_splits(\n split_0: dsl.InputPath('MaterializedSplit'),\n\ + \ split_1: dsl.InputPath('MaterializedSplit'),\n splits: dsl.OutputPath('MaterializedSplit'),\n\ + ):\n \"\"\"Merge two materialized splits.\n\n Args:\n split_0: The\ + \ first materialized split.\n split_1: The second materialized split.\n\ + \ splits: The merged materialized split.\n \"\"\"\n with open(split_0,\ + \ 'r') as f:\n split_0_content = f.read()\n with open(split_1, 'r')\ + \ as f:\n split_1_content = f.read()\n with open(splits, 'w') as f:\n\ + \ f.write(','.join([split_0_content, split_1_content]))\n\n" + image: python:3.7 + exec-model-batch-explanation: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-explanation-3: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"model_parameters\": ", "{{$.inputs.parameters[''model_parameters'']}}", + ", \"output_config\": {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.13 + exec-model-batch-predict: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-2: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-3: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-4: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-batch-predict-5: + container: + args: + - --type + - BatchPredictionJob + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''job_display_name'']}}", + "\", ", {"IfPresent": {"InputName": "model", "Then": {"Concat": ["\"model\": + \"", "{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}", "\","]}}}, + " \"input_config\": {", "\"instances_format\": \"", "{{$.inputs.parameters[''instances_format'']}}", + "\"", ", \"gcs_source\": {", "\"uris\":", "{{$.inputs.parameters[''gcs_source_uris'']}}", + "}", ", \"bigquery_source\": {", "\"input_uri\": \"", "{{$.inputs.parameters[''bigquery_source_input_uri'']}}", + "\"", "}", "}", ", \"instance_config\": {", "\"instance_type\": \"", "{{$.inputs.parameters[''instance_type'']}}", + "\"", ", \"key_field\": \"", "{{$.inputs.parameters[''key_field'']}}", "\" + ", {"IfPresent": {"InputName": "included_fields", "Then": {"Concat": [", + \"included_fields\": ", "{{$.inputs.parameters[''included_fields'']}}"]}}}, + {"IfPresent": {"InputName": "excluded_fields", "Then": {"Concat": [", \"excluded_fields\": + ", "{{$.inputs.parameters[''excluded_fields'']}}"]}}}, "}", ", \"model_parameters\": + ", "{{$.inputs.parameters[''model_parameters'']}}", ", \"output_config\": + {", "\"predictions_format\": \"", "{{$.inputs.parameters[''predictions_format'']}}", + "\"", ", \"gcs_destination\": {", "\"output_uri_prefix\": \"", "{{$.inputs.parameters[''gcs_destination_output_uri_prefix'']}}", + "\"", "}", ", \"bigquery_destination\": {", "\"output_uri\": \"", "{{$.inputs.parameters[''bigquery_destination_output_uri'']}}", + "\"", "}", "}", ", \"dedicated_resources\": {", "\"machine_spec\": {", "\"machine_type\": + \"", "{{$.inputs.parameters[''machine_type'']}}", "\"", ", \"accelerator_type\": + \"", "{{$.inputs.parameters[''accelerator_type'']}}", "\"", ", \"accelerator_count\": + ", "{{$.inputs.parameters[''accelerator_count'']}}", "}", ", \"starting_replica_count\": + ", "{{$.inputs.parameters[''starting_replica_count'']}}", ", \"max_replica_count\": + ", "{{$.inputs.parameters[''max_replica_count'']}}", "}", ", \"manual_batch_tuning_parameters\": + {", "\"batch_size\": ", "{{$.inputs.parameters[''manual_batch_tuning_parameters_batch_size'']}}", + "}", ", \"generate_explanation\": ", "{{$.inputs.parameters[''generate_explanation'']}}", + ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", ", \"encryption_spec\": + {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-2: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-3: + container: + args: + - --setup_file + - /setup.py + - --json_mode + - 'true' + - --project_id + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --problem_type + - '{{$.inputs.parameters[''problem_type'']}}' + - --batch_prediction_format + - '{{$.inputs.parameters[''predictions_format'']}}' + - --batch_prediction_gcs_source + - '{{$.inputs.artifacts[''batch_prediction_job''].metadata[''gcsOutputDirectory'']}}' + - --ground_truth_format + - '{{$.inputs.parameters[''ground_truth_format'']}}' + - --key_prefix_in_prediction_dataset + - instance + - --root_dir + - '{{$.inputs.parameters[''root_dir'']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}' + - --classification_type + - multiclass + - --ground_truth_column + - instance.{{$.inputs.parameters['ground_truth_column']}} + - --prediction_score_column + - '{{$.inputs.parameters[''prediction_score_column'']}}' + - --prediction_label_column + - '{{$.inputs.parameters[''prediction_label_column'']}}' + - --prediction_id_column + - '' + - --example_weight_column + - '' + - --generate_feature_attribution + - 'false' + - --dataflow_job_prefix + - evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}} + - --dataflow_service_account + - '{{$.inputs.parameters[''dataflow_service_account'']}}' + - --dataflow_disk_size + - '{{$.inputs.parameters[''dataflow_disk_size'']}}' + - --dataflow_machine_type + - '{{$.inputs.parameters[''dataflow_machine_type'']}}' + - --dataflow_workers_num + - '{{$.inputs.parameters[''dataflow_workers_num'']}}' + - --dataflow_max_workers_num + - '{{$.inputs.parameters[''dataflow_max_workers_num'']}}' + - --dataflow_subnetwork + - '{{$.inputs.parameters[''dataflow_subnetwork'']}}' + - --dataflow_use_public_ips + - '{{$.inputs.parameters[''dataflow_use_public_ips'']}}' + - --kms_key_name + - '{{$.inputs.parameters[''encryption_spec_key_name'']}}' + - --output_metrics_gcs_path + - '{{$.outputs.artifacts[''evaluation_metrics''].uri}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + command: + - python + - /main.py + image: gcr.io/ml-pipeline/model-evaluation:v0.4 + exec-model-evaluation-import: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-2: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-evaluation-import-3: + container: + args: + - '{"IfPresent": {"InputName": "metrics", "Then": ["--metrics", "{{$.inputs.artifacts[''metrics''].uri}}", + "--metrics_explanation", "{{$.inputs.artifacts[''metrics''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "explanation", "Then": ["--explanation", "{{$.inputs.artifacts[''explanation''].metadata[''explanation_gcs_path'']}}"]}}' + - '{"IfPresent": {"InputName": "classification_metrics", "Then": ["--classification_metrics", + "{{$.inputs.artifacts[''classification_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "forecasting_metrics", "Then": ["--forecasting_metrics", + "{{$.inputs.artifacts[''forecasting_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "regression_metrics", "Then": ["--regression_metrics", + "{{$.inputs.artifacts[''regression_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "text_generation_metrics", "Then": ["--text_generation_metrics", + "{{$.inputs.artifacts[''text_generation_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "question_answering_metrics", "Then": ["--question_answering_metrics", + "{{$.inputs.artifacts[''question_answering_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "summarization_metrics", "Then": ["--summarization_metrics", + "{{$.inputs.artifacts[''summarization_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "feature_attributions", "Then": ["--feature_attributions", + "{{$.inputs.artifacts[''feature_attributions''].uri}}"]}}' + - '{"IfPresent": {"InputName": "embedding_metrics", "Then": ["--embedding_metrics", + "{{$.inputs.artifacts[''embedding_metrics''].uri}}"]}}' + - '{"IfPresent": {"InputName": "problem_type", "Then": ["--problem_type", + "{{$.inputs.parameters[''problem_type'']}}"]}}' + - --display_name + - '{{$.inputs.parameters[''display_name'']}}' + - --dataset_path + - '{{$.inputs.parameters[''dataset_path'']}}' + - --dataset_paths + - '{{$.inputs.parameters[''dataset_paths'']}}' + - --dataset_type + - '{{$.inputs.parameters[''dataset_type'']}}' + - --pipeline_job_id + - '{{$.pipeline_job_uuid}}' + - --pipeline_job_resource_name + - '{{$.pipeline_job_resource_name}}' + - --model_name + - '{{$.inputs.artifacts[''model''].metadata[''resourceName'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --evaluation_resource_name + - '{{$.outputs.parameters[''evaluation_resource_name''].output_file}}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container._implementation.model_evaluation.import_model_evaluation + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1 + exec-model-upload: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-2: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-model-upload-3: + container: + args: + - --type + - UploadModel + - --payload + - '{"Concat": ["{", "\"display_name\": \"", "{{$.inputs.parameters[''display_name'']}}", + "\"", ", \"description\": \"", "{{$.inputs.parameters[''description'']}}", + "\"", ", \"explanation_spec\": {", "\"parameters\": ", "{{$.inputs.parameters[''explanation_parameters'']}}", + ", \"metadata\": ", "{{$.inputs.parameters[''explanation_metadata'']}}", + "}", ", \"explanation_metadata_artifact\": \"", "{{$.inputs.artifacts[''explanation_metadata_artifact''].uri}}", + "\"", ", \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}", ", \"labels\": ", "{{$.inputs.parameters[''labels'']}}", "}"]}' + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --executor_input + - '{{$}}' + - '{"IfPresent": {"InputName": "parent_model", "Then": ["--parent_model_name", + "{{$.inputs.artifacts[''parent_model''].metadata[''resourceName'']}}"]}}' + command: + - python3 + - -u + - -m + - launcher + image: gcr.io/ml-pipeline/automl-tables-private:1.0.15 + exec-read-input-uri: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _read_input_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _read_input_uri(\n split_uri: dsl.InputPath('Dataset'),\n\ + ) -> list: # Required by KFP; pylint:disable=g-bare-generic\n \"\"\"Construct\ + \ Dataset based on the batch prediction job.\n\n Args:\n split_uri:\ + \ Tbe path to the file that contains Dataset data.\n\n Returns:\n The\ + \ list of string that represents the batch prediction input files.\n \"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ + \ return data_source['tf_record_data_source']['file_patterns']\n\n" + image: python:3.7 + exec-read-input-uri-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _read_input_uri + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _read_input_uri(\n split_uri: dsl.InputPath('Dataset'),\n\ + ) -> list: # Required by KFP; pylint:disable=g-bare-generic\n \"\"\"Construct\ + \ Dataset based on the batch prediction job.\n\n Args:\n split_uri:\ + \ Tbe path to the file that contains Dataset data.\n\n Returns:\n The\ + \ list of string that represents the batch prediction input files.\n \"\ + \"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n\ + \ return data_source['tf_record_data_source']['file_patterns']\n\n" + image: python:3.7 + exec-set-optional-inputs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _set_optional_inputs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _set_optional_inputs(\n project: str,\n location: str,\n\ + \ data_source_csv_filenames: str,\n data_source_bigquery_table_path:\ + \ str,\n vertex_dataset: dsl.Input[dsl.Artifact],\n model_display_name:\ + \ str,\n) -> NamedTuple(\n 'Outputs',\n [\n ('data_source_csv_filenames',\ + \ str),\n ('data_source_bigquery_table_path', str),\n ('model_display_name',\ + \ str),\n ],\n):\n \"\"\"Get the data source URI.\n\n Args:\n project:\ + \ The GCP project that runs the pipeline components.\n location: The\ + \ GCP region that runs the pipeline components.\n data_source_csv_filenames:\ + \ The CSV GCS path when data source is CSV.\n data_source_bigquery_table_path:\ + \ The BigQuery table when data source is BQ.\n vertex_dataset: The Vertex\ + \ dataset when data source is Vertex dataset.\n model_display_name: The\ + \ uploaded model's display name.\n\n Returns:\n A named tuple of CSV\ + \ or BQ URI.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \ import collections\n from google.cloud import aiplatform\n from google.cloud\ + \ import aiplatform_v1beta1 as aip\n import uuid\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name\n\ + \n if not model_display_name:\n model_display_name = f'tabular-workflow-model-{uuid.uuid4()}'\n\ + \n if vertex_dataset is not None:\n # of format\n # projects/294348452381/locations/us-central1/datasets/7104764862735056896\n\ + \ dataset_name = vertex_dataset.metadata['resourceName']\n\n aiplatform.init(project=project,\ + \ location=location)\n client = aip.DatasetServiceClient(\n client_options={'api_endpoint':\ + \ f'{location}-aiplatform.googleapis.com'}\n )\n dataset = client.get_dataset(name=dataset_name)\n\ + \ input_config = dataset.metadata['inputConfig']\n if 'gcsSource'\ + \ in input_config:\n data_source_csv_filenames = ','.join(input_config['gcsSource']['uri'])\n\ + \ elif 'bigquerySource' in input_config:\n data_source_bigquery_table_path\ + \ = input_config['bigquerySource']['uri']\n elif data_source_csv_filenames:\n\ + \ pass\n elif data_source_bigquery_table_path:\n pass\n else:\n\ + \ raise ValueError(\n 'One of vertex_dataset, data_source_csv_filenames,'\n\ + \ ' data_source_bigquery_table_path must be specified'\n )\n\n\ + \ return collections.namedtuple(\n 'Outputs',\n [\n \ + \ 'data_source_csv_filenames',\n 'data_source_bigquery_table_path',\n\ + \ 'model_display_name',\n ],\n )(\n data_source_csv_filenames,\n\ + \ data_source_bigquery_table_path,\n model_display_name,\n )\n\ + \n" + image: us-docker.pkg.dev/vertex-ai/automl-tabular/kfp-v2-base:20231002_0125 + exec-string-not-empty: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _string_not_empty + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _string_not_empty(value: str) -> str:\n \"\"\"Check if the input\ + \ string value is not empty.\n\n Args:\n value: String value to be checked.\n\ + \n Returns:\n Boolean value. -> 'true' if empty, 'false' if not empty.\ + \ We need to use str\n instead of bool due to a limitation in KFP compiler.\n\ + \ \"\"\"\n return 'true' if value else 'false'\n\n" + image: python:3.7 + exec-tabular-stats-and-example-gen: + container: + args: + - --type + - CustomJob + - --project + - '{{$.inputs.parameters[''project'']}}' + - --location + - '{{$.inputs.parameters[''location'']}}' + - --gcp_resources + - '{{$.outputs.parameters[''gcp_resources''].output_file}}' + - --payload + - '{"Concat": ["{\"display_name\": \"tabular-stats-and-example-gen-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", + \"encryption_spec\": {\"kms_key_name\":\"", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": + {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"", + "us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125", "\", + \"args\": [\"stats_generator\",", "\"--train_spec={\\\"prediction_type\\\": + \\\"", "{{$.inputs.parameters[''prediction_type'']}}", "\\\", \\\"target_column\\\": + \\\"", "{{$.inputs.parameters[''target_column_name'']}}", "\\\", \\\"optimization_objective\\\": + \\\"", "{{$.inputs.parameters[''optimization_objective'']}}", "\\\", \\\"weight_column_name\\\": + \\\"", "{{$.inputs.parameters[''weight_column_name'']}}", "\\\", \\\"transformations\\\": + ", "{{$.inputs.parameters[''transformations'']}}", ", \\\"quantiles\\\": + ", "{{$.inputs.parameters[''quantiles'']}}", ", \\\"enable_probabilistic_inference\\\": + ", "{{$.inputs.parameters[''enable_probabilistic_inference'']}}", "}\", + \"--transformations_override_path=", "{{$.inputs.parameters[''transformations_path'']}}", + "\", \"--data_source_csv_filenames=", "{{$.inputs.parameters[''data_source_csv_filenames'']}}", + "\", \"--data_source_bigquery_table_path=", "{{$.inputs.parameters[''data_source_bigquery_table_path'']}}", + "\", \"--predefined_split_key=", "{{$.inputs.parameters[''predefined_split_key'']}}", + "\", \"--timestamp_split_key=", "{{$.inputs.parameters[''timestamp_split_key'']}}", + "\", \"--stratified_split_key=", "{{$.inputs.parameters[''stratified_split_key'']}}", + "\", \"--training_fraction=", "{{$.inputs.parameters[''training_fraction'']}}", + "\", \"--validation_fraction=", "{{$.inputs.parameters[''validation_fraction'']}}", + "\", \"--test_fraction=", "{{$.inputs.parameters[''test_fraction'']}}", + "\", \"--target_column=", "{{$.inputs.parameters[''target_column_name'']}}", + "\", \"--request_type=", "{{$.inputs.parameters[''request_type'']}}", "\", + \"--optimization_objective_recall_value=", "{{$.inputs.parameters[''optimization_objective_recall_value'']}}", + "\", \"--optimization_objective_precision_value=", "{{$.inputs.parameters[''optimization_objective_precision_value'']}}", + "\", \"--example_gen_gcs_output_prefix=", "{{$.inputs.parameters[''root_dir'']}}", + "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/example_gen_output\", + \"--dataset_stats_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/stats/\", + \"--stats_result_path=", "{{$.outputs.artifacts[''dataset_stats''].uri}}", + "\", \"--dataset_schema_path=", "{{$.outputs.artifacts[''dataset_schema''].uri}}", + "\", \"--job_name=tabular-stats-and-example-gen-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "\", \"--dataflow_project=", "{{$.inputs.parameters[''project'']}}", "\", + \"--error_file_path=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", + \"--dataflow_staging_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", + \"--dataflow_tmp_dir=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", + \"--dataflow_max_num_workers=", "{{$.inputs.parameters[''dataflow_max_num_workers'']}}", + "\", \"--dataflow_worker_container_image=", "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125", + "\", \"--dataflow_machine_type=", "{{$.inputs.parameters[''dataflow_machine_type'']}}", + "\", \"--dataflow_disk_size_gb=", "{{$.inputs.parameters[''dataflow_disk_size_gb'']}}", + "\", \"--dataflow_kms_key=", "{{$.inputs.parameters[''encryption_spec_key_name'']}}", + "\", \"--dataflow_subnetwork_fully_qualified=", "{{$.inputs.parameters[''dataflow_subnetwork'']}}", + "\", \"--dataflow_use_public_ips=", "{{$.inputs.parameters[''dataflow_use_public_ips'']}}", + "\", \"--dataflow_service_account=", "{{$.inputs.parameters[''dataflow_service_account'']}}", + "\", \"--is_distill=", "{{$.inputs.parameters[''run_distillation'']}}", + "\", \"--additional_experiments=", "{{$.inputs.parameters[''additional_experiments'']}}", + "\", \"--metadata_path=", "{{$.outputs.artifacts[''metadata''].uri}}", "\", + \"--train_split=", "{{$.outputs.artifacts[''train_split''].uri}}", "\", + \"--eval_split=", "{{$.outputs.artifacts[''eval_split''].uri}}", "\", \"--test_split=", + "{{$.outputs.artifacts[''test_split''].uri}}", "\", \"--test_split_for_batch_prediction_component=", + "{{$.outputs.parameters[''test_split_json''].output_file}}", "\", \"--downsampled_test_split_for_batch_prediction_component=", + "{{$.outputs.parameters[''downsampled_test_split_json''].output_file}}", + "\", \"--instance_baseline_path=", "{{$.outputs.artifacts[''instance_baseline''].uri}}", + "\", \"--lro_job_info=", "{{$.inputs.parameters[''root_dir'']}}", "/{{$.pipeline_job_uuid}}/lro\", + \"--gcp_resources_path=", "{{$.outputs.parameters[''gcp_resources''].output_file}}", + "\", \"--parse_json=true\", \"--generate_additional_downsample_test_split=true\", + \"--executor_input={{$.json_escape[1]}}\"]}}]}}"]}' + command: + - python3 + - -u + - -m + - google_cloud_pipeline_components.container.v1.custom_job.launcher + image: gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44 + exec-write-bp-result-path: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _write_bp_result_path + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _write_bp_result_path(\n bp_job: dsl.Input[dsl.Artifact],\n\ + \ result: dsl.OutputPath('Dataset'),\n):\n \"\"\"Construct Dataset based\ + \ on the batch prediction job.\n\n Args:\n bp_job: The batch prediction\ + \ job artifact.\n result: Tbe path to the file that contains Dataset\ + \ data.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ directory = bp_job.metadata['gcsOutputDirectory']\n data_source = {\n\ + \ 'tf_record_data_source': {\n 'file_patterns': [\n \ + \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ + \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ + \n" + image: python:3.7 + exec-write-bp-result-path-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - _write_bp_result_path + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-rc.2'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef _write_bp_result_path(\n bp_job: dsl.Input[dsl.Artifact],\n\ + \ result: dsl.OutputPath('Dataset'),\n):\n \"\"\"Construct Dataset based\ + \ on the batch prediction job.\n\n Args:\n bp_job: The batch prediction\ + \ job artifact.\n result: Tbe path to the file that contains Dataset\ + \ data.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\ + \ directory = bp_job.metadata['gcsOutputDirectory']\n data_source = {\n\ + \ 'tf_record_data_source': {\n 'file_patterns': [\n \ + \ f'{directory}/prediction.results-*',\n ],\n 'coder':\ + \ 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\ + \n" + image: python:3.7 +pipelineInfo: + description: 'Complete AutoML Tables pipeline. + + Includes feature engineering, architecture search, and hyper-parameter tuning.' + name: automl-tabular +root: + dag: + outputs: + artifacts: + feature-attribution-2-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-2-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-3-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-3-feature_attributions + producerSubtask: exit-handler-1 + feature-attribution-feature_attributions: + artifactSelectors: + - outputArtifactKey: feature-attribution-feature_attributions + producerSubtask: exit-handler-1 + model-evaluation-2-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-2-evaluation_metrics + producerSubtask: exit-handler-1 + model-evaluation-3-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-3-evaluation_metrics + producerSubtask: exit-handler-1 + model-evaluation-evaluation_metrics: + artifactSelectors: + - outputArtifactKey: model-evaluation-evaluation_metrics + producerSubtask: exit-handler-1 + tasks: + automl-tabular-finalizer: + cachingOptions: + enableCache: true + componentRef: + name: comp-automl-tabular-finalizer + dependentTasks: + - exit-handler-1 + inputs: + parameters: + location: + componentInputParameter: location + project: + componentInputParameter: project + root_dir: + componentInputParameter: root_dir + taskInfo: + name: automl-tabular-finalizer + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + dependentTasks: + - set-optional-inputs + inputs: + artifacts: + pipelinechannel--parent_model: + componentInputArtifact: parent_model + parameters: + pipelinechannel--additional_experiments: + componentInputParameter: additional_experiments + pipelinechannel--cv_trainer_worker_pool_specs_override: + componentInputParameter: cv_trainer_worker_pool_specs_override + pipelinechannel--dataflow_service_account: + componentInputParameter: dataflow_service_account + pipelinechannel--dataflow_subnetwork: + componentInputParameter: dataflow_subnetwork + pipelinechannel--dataflow_use_public_ips: + componentInputParameter: dataflow_use_public_ips + pipelinechannel--disable_early_stopping: + componentInputParameter: disable_early_stopping + pipelinechannel--distill_batch_predict_machine_type: + componentInputParameter: distill_batch_predict_machine_type + pipelinechannel--distill_batch_predict_max_replica_count: + componentInputParameter: distill_batch_predict_max_replica_count + pipelinechannel--distill_batch_predict_starting_replica_count: + componentInputParameter: distill_batch_predict_starting_replica_count + pipelinechannel--enable_probabilistic_inference: + componentInputParameter: enable_probabilistic_inference + pipelinechannel--encryption_spec_key_name: + componentInputParameter: encryption_spec_key_name + pipelinechannel--evaluation_batch_explain_machine_type: + componentInputParameter: evaluation_batch_explain_machine_type + pipelinechannel--evaluation_batch_explain_max_replica_count: + componentInputParameter: evaluation_batch_explain_max_replica_count + pipelinechannel--evaluation_batch_explain_starting_replica_count: + componentInputParameter: evaluation_batch_explain_starting_replica_count + pipelinechannel--evaluation_batch_predict_machine_type: + componentInputParameter: evaluation_batch_predict_machine_type + pipelinechannel--evaluation_batch_predict_max_replica_count: + componentInputParameter: evaluation_batch_predict_max_replica_count + pipelinechannel--evaluation_batch_predict_starting_replica_count: + componentInputParameter: evaluation_batch_predict_starting_replica_count + pipelinechannel--evaluation_dataflow_disk_size_gb: + componentInputParameter: evaluation_dataflow_disk_size_gb + pipelinechannel--evaluation_dataflow_machine_type: + componentInputParameter: evaluation_dataflow_machine_type + pipelinechannel--evaluation_dataflow_max_num_workers: + componentInputParameter: evaluation_dataflow_max_num_workers + pipelinechannel--evaluation_dataflow_starting_num_workers: + componentInputParameter: evaluation_dataflow_starting_num_workers + pipelinechannel--export_additional_model_without_custom_ops: + componentInputParameter: export_additional_model_without_custom_ops + pipelinechannel--fast_testing: + componentInputParameter: fast_testing + pipelinechannel--location: + componentInputParameter: location + pipelinechannel--model_description: + componentInputParameter: model_description + pipelinechannel--optimization_objective: + componentInputParameter: optimization_objective + pipelinechannel--optimization_objective_precision_value: + componentInputParameter: optimization_objective_precision_value + pipelinechannel--optimization_objective_recall_value: + componentInputParameter: optimization_objective_recall_value + pipelinechannel--predefined_split_key: + componentInputParameter: predefined_split_key + pipelinechannel--prediction_type: + componentInputParameter: prediction_type + pipelinechannel--project: + componentInputParameter: project + pipelinechannel--quantiles: + componentInputParameter: quantiles + pipelinechannel--root_dir: + componentInputParameter: root_dir + pipelinechannel--run_distillation: + componentInputParameter: run_distillation + pipelinechannel--run_evaluation: + componentInputParameter: run_evaluation + pipelinechannel--set-optional-inputs-data_source_bigquery_table_path: + taskOutputParameter: + outputParameterKey: data_source_bigquery_table_path + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-data_source_csv_filenames: + taskOutputParameter: + outputParameterKey: data_source_csv_filenames + producerTask: set-optional-inputs + pipelinechannel--set-optional-inputs-model_display_name: + taskOutputParameter: + outputParameterKey: model_display_name + producerTask: set-optional-inputs + pipelinechannel--stage_1_num_parallel_trials: + componentInputParameter: stage_1_num_parallel_trials + pipelinechannel--stage_1_tuner_worker_pool_specs_override: + componentInputParameter: stage_1_tuner_worker_pool_specs_override + pipelinechannel--stage_1_tuning_result_artifact_uri: + componentInputParameter: stage_1_tuning_result_artifact_uri + pipelinechannel--stage_2_num_parallel_trials: + componentInputParameter: stage_2_num_parallel_trials + pipelinechannel--stage_2_num_selected_trials: + componentInputParameter: stage_2_num_selected_trials + pipelinechannel--stats_and_example_gen_dataflow_disk_size_gb: + componentInputParameter: stats_and_example_gen_dataflow_disk_size_gb + pipelinechannel--stats_and_example_gen_dataflow_machine_type: + componentInputParameter: stats_and_example_gen_dataflow_machine_type + pipelinechannel--stats_and_example_gen_dataflow_max_num_workers: + componentInputParameter: stats_and_example_gen_dataflow_max_num_workers + pipelinechannel--stratified_split_key: + componentInputParameter: stratified_split_key + pipelinechannel--study_spec_parameters_override: + componentInputParameter: study_spec_parameters_override + pipelinechannel--target_column: + componentInputParameter: target_column + pipelinechannel--test_fraction: + componentInputParameter: test_fraction + pipelinechannel--timestamp_split_key: + componentInputParameter: timestamp_split_key + pipelinechannel--train_budget_milli_node_hours: + componentInputParameter: train_budget_milli_node_hours + pipelinechannel--training_fraction: + componentInputParameter: training_fraction + pipelinechannel--transform_dataflow_disk_size_gb: + componentInputParameter: transform_dataflow_disk_size_gb + pipelinechannel--transform_dataflow_machine_type: + componentInputParameter: transform_dataflow_machine_type + pipelinechannel--transform_dataflow_max_num_workers: + componentInputParameter: transform_dataflow_max_num_workers + pipelinechannel--transformations: + componentInputParameter: transformations + pipelinechannel--validation_fraction: + componentInputParameter: validation_fraction + pipelinechannel--weight_column: + componentInputParameter: weight_column + taskInfo: + name: exit-handler-1 + set-optional-inputs: + cachingOptions: + enableCache: true + componentRef: + name: comp-set-optional-inputs + inputs: + artifacts: + vertex_dataset: + componentInputArtifact: vertex_dataset + parameters: + data_source_bigquery_table_path: + componentInputParameter: data_source_bigquery_table_path + data_source_csv_filenames: + componentInputParameter: data_source_csv_filenames + location: + componentInputParameter: location + model_display_name: + componentInputParameter: model_display_name + project: + componentInputParameter: project + taskInfo: + name: set-optional-inputs + inputDefinitions: + artifacts: + parent_model: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Vertex Model to upload this model as a version of. + isOptional: true + vertex_dataset: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: The Vertex dataset artifact. + parameters: + additional_experiments: + description: Use this field to config private preview features. + isOptional: true + parameterType: STRUCT + cv_trainer_worker_pool_specs_override: + description: 'The dictionary for overriding stage + + cv trainer worker pool spec.' + isOptional: true + parameterType: LIST + data_source_bigquery_table_path: + defaultValue: '' + description: 'The BigQuery table path of format + + bq://bq_project.bq_dataset.bq_table' + isOptional: true + parameterType: STRING + data_source_csv_filenames: + defaultValue: '' + description: 'A string that represents a list of comma + + separated CSV filenames.' + isOptional: true + parameterType: STRING + dataflow_service_account: + defaultValue: '' + description: Custom service account to run dataflow jobs. + isOptional: true + parameterType: STRING + dataflow_subnetwork: + defaultValue: '' + description: 'Dataflow''s fully qualified subnetwork name, when empty + + the default subnetwork will be used. Example: + + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications' + isOptional: true + parameterType: STRING + dataflow_use_public_ips: + defaultValue: true + description: 'Specifies whether Dataflow workers use public IP + + addresses.' + isOptional: true + parameterType: BOOLEAN + disable_early_stopping: + defaultValue: false + description: If disable easly stopping. + isOptional: true + parameterType: BOOLEAN + distill_batch_predict_machine_type: + defaultValue: n1-standard-16 + description: 'The prediction server machine type for + + batch predict component in the model distillation.' + isOptional: true + parameterType: STRING + distill_batch_predict_max_replica_count: + defaultValue: 25.0 + description: 'The max number of prediction server + + for batch predict component in the model distillation.' + isOptional: true + parameterType: NUMBER_INTEGER + distill_batch_predict_starting_replica_count: + defaultValue: 25.0 + description: 'The initial number of + + prediction server for batch predict component in the model distillation.' + isOptional: true + parameterType: NUMBER_INTEGER + enable_probabilistic_inference: + defaultValue: false + description: 'If probabilistic inference is enabled, the + + model will fit a distribution that captures the uncertainty of a + + prediction. At inference time, the predictive distribution is used to make + + a point prediction that minimizes the optimization objective. For example, + + the mean of a predictive distribution is the point prediction that + + minimizes RMSE loss. If quantiles are specified, then the quantiles of the + + distribution are also returned.' + isOptional: true + parameterType: BOOLEAN + encryption_spec_key_name: + defaultValue: '' + description: The KMS key name. + isOptional: true + parameterType: STRING + evaluation_batch_explain_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch explain components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_explain_max_replica_count: + defaultValue: 10.0 + description: 'The max number of prediction + + server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_explain_starting_replica_count: + defaultValue: 10.0 + description: 'The initial number of + + prediction server for batch explain components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_machine_type: + defaultValue: n1-highmem-8 + description: 'The prediction server machine type + + for batch predict components during evaluation.' + isOptional: true + parameterType: STRING + evaluation_batch_predict_max_replica_count: + defaultValue: 20.0 + description: 'The max number of prediction + + server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_batch_predict_starting_replica_count: + defaultValue: 20.0 + description: 'The initial number of + + prediction server for batch predict components during evaluation.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_disk_size_gb: + defaultValue: 50.0 + description: 'Dataflow worker''s disk size in GB for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_machine_type: + defaultValue: n1-standard-4 + description: 'The dataflow machine type for evaluation + + components.' + isOptional: true + parameterType: STRING + evaluation_dataflow_max_num_workers: + defaultValue: 100.0 + description: 'The max number of Dataflow workers for + + evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + evaluation_dataflow_starting_num_workers: + defaultValue: 10.0 + description: 'The initial number of Dataflow + + workers for evaluation components.' + isOptional: true + parameterType: NUMBER_INTEGER + export_additional_model_without_custom_ops: + defaultValue: false + description: 'Whether to export additional + + model without custom TensorFlow operators.' + isOptional: true + parameterType: BOOLEAN + fast_testing: + defaultValue: false + description: Internal flag used for presubmit tests. + isOptional: true + parameterType: BOOLEAN + location: + description: The GCP region that runs the pipeline components. + parameterType: STRING + model_description: + defaultValue: '' + description: The description name of the uploaded Vertex model, + isOptional: true + parameterType: STRING + model_display_name: + defaultValue: '' + description: The display name of the uploaded Vertex model, + isOptional: true + parameterType: STRING + optimization_objective: + description: 'For binary classification, "maximize-au-roc", + + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", + or + + "maximize-recall-at-precision". For multi class classification, + + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + + "minimize-rmsle".' + parameterType: STRING + optimization_objective_precision_value: + defaultValue: -1.0 + description: 'Required when optimization_objective + + is ''maximize-recall-at-precision''. Must be between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + optimization_objective_recall_value: + defaultValue: -1.0 + description: 'Required when optimization_objective is + + ''maximize-precision-at-recall''. Must be between 0 and 1, inclusive.' + isOptional: true + parameterType: NUMBER_DOUBLE + predefined_split_key: + defaultValue: '' + description: The predefined_split column name. + isOptional: true + parameterType: STRING + prediction_type: + description: 'The type of prediction the model is to produce. + + "classification" or "regression".' + parameterType: STRING + project: + description: The GCP project that runs the pipeline components. + parameterType: STRING + quantiles: + description: 'Quantiles to use for probabilistic inference. Up to 5 quantiles + + are allowed of values between 0 and 1, exclusive. Represents the quantiles + + to use for that objective. Quantiles must be unique.' + isOptional: true + parameterType: LIST + root_dir: + description: The root GCS directory for the pipeline components. + parameterType: STRING + run_distillation: + defaultValue: false + description: 'Whether the distillation should be applied to the + + training.' + isOptional: true + parameterType: BOOLEAN + run_evaluation: + defaultValue: false + description: Whether to run evaluation steps during training. + isOptional: true + parameterType: BOOLEAN + stage_1_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 1. + isOptional: true + parameterType: NUMBER_INTEGER + stage_1_tuner_worker_pool_specs_override: + description: 'The dictionary for overriding + + stage 1 tuner worker pool spec.' + isOptional: true + parameterType: LIST + stage_1_tuning_result_artifact_uri: + defaultValue: '' + description: 'The stage 1 tuning result artifact GCS + + URI.' + isOptional: true + parameterType: STRING + stage_2_num_parallel_trials: + defaultValue: 35.0 + description: Number of parallel trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stage_2_num_selected_trials: + defaultValue: 5.0 + description: Number of selected trails for stage 2. + isOptional: true + parameterType: NUMBER_INTEGER + stats_and_example_gen_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in + + GB for stats_and_example_gen component.' + isOptional: true + parameterType: NUMBER_INTEGER + stats_and_example_gen_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for + + stats_and_example_gen component.' + isOptional: true + parameterType: STRING + stats_and_example_gen_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow + + workers for stats_and_example_gen component.' + isOptional: true + parameterType: NUMBER_INTEGER + stratified_split_key: + defaultValue: '' + description: The stratified_split column name. + isOptional: true + parameterType: STRING + study_spec_parameters_override: + description: The list for overriding study spec. + isOptional: true + parameterType: LIST + target_column: + description: The target column name. + parameterType: STRING + test_fraction: + defaultValue: -1.0 + description: float = The test fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + timestamp_split_key: + defaultValue: '' + description: The timestamp_split column name. + isOptional: true + parameterType: STRING + train_budget_milli_node_hours: + description: 'The train budget of creating this model, + + expressed in milli node hours i.e. 1,000 value in this field means 1 node + + hour.' + parameterType: NUMBER_DOUBLE + training_fraction: + defaultValue: -1.0 + description: The training fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + transform_dataflow_disk_size_gb: + defaultValue: 40.0 + description: 'Dataflow worker''s disk size in GB for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transform_dataflow_machine_type: + defaultValue: n1-standard-16 + description: 'The dataflow machine type for transform + + component.' + isOptional: true + parameterType: STRING + transform_dataflow_max_num_workers: + defaultValue: 25.0 + description: 'The max number of Dataflow workers for + + transform component.' + isOptional: true + parameterType: NUMBER_INTEGER + transformations: + description: 'The path to a GCS file containing the transformations to + + apply.' + parameterType: STRING + validation_fraction: + defaultValue: -1.0 + description: The validation fraction. + isOptional: true + parameterType: NUMBER_DOUBLE + weight_column: + defaultValue: '' + description: The weight column name. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + feature-attribution-2-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-3-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + feature-attribution-feature_attributions: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-2-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-3-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 + model-evaluation-evaluation_metrics: + artifactType: + schemaTitle: system.Metrics + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py new file mode 100644 index 0000000000..fc415969ff --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/cv_trainer.py @@ -0,0 +1,160 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Tabular Cross Validation Trainer component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.container_component +def automl_tabular_cv_trainer( + project: str, + location: str, + root_dir: str, + deadline_hours: float, + num_parallel_trials: int, + single_run_max_secs: int, + num_selected_trials: int, + transform_output: Input[Artifact], + metadata: Input[Artifact], + materialized_cv_splits: Input[Artifact], + tuning_result_input: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + tuning_result_output: Output[Artifact], + execution_metrics: dsl.OutputPath(dict), + worker_pool_specs_override_json: Optional[list] = [], + num_selected_features: Optional[int] = 0, + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Tunes AutoML Tabular models and selects top trials using cross-validation. + + Args: + project: Project to run Cross-validation trainer. + location: Location for running the Cross-validation trainer. + root_dir: The Cloud Storage location to store the output. + worker_pool_specs_override_json: JSON worker pool specs. E.g., [{"machine_spec": {"machine_type": "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": "n1-standard-16"}}] + deadline_hours: Number of hours the cross-validation trainer should run. + num_parallel_trials: Number of parallel training trials. + single_run_max_secs: Max number of seconds each training trial runs. + num_selected_trials: Number of selected trials. The number of weak learners in the final model is 5 * num_selected_trials. + num_selected_features: Number of selected features. The number of features to learn in the NN models. + transform_output: The transform output artifact. + metadata: The tabular example gen metadata. + materialized_cv_splits: The materialized cross-validation splits. + tuning_result_input: AutoML Tabular tuning result. + encryption_spec_key_name: Customer-managed encryption key. + + Returns: + tuning_result_output: The trained model and architectures. + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + execution_metrics: Core metrics in dictionary of component execution. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "automl-tabular-cv-tuner-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "args": ["l2l_cv_tuner", "--transform_output_path=', + transform_output.uri, + '", "--training_docker_uri=', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + ( + f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "--training_base_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train",' + ' "--num_parallel_trial=' + ), + num_parallel_trials, + '", "--single_run_max_secs=', + single_run_max_secs, + '", "--deadline_hours=', + deadline_hours, + ( + '", "--valid_trials_completed_threshold=0.7",' + ' "--num_selected_trials=' + ), + num_selected_trials, + '", "--num_selected_features=', + num_selected_features, + '", "--lro_job_info=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/lro",' + ' "--error_file_path=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb",' + ' "--metadata_path=' + ), + metadata.uri, + '", "--materialized_cv_splits=', + materialized_cv_splits.uri, + '", "--tuning_result_input_path=', + tuning_result_input.uri, + '", "--tuning_result_output_path=', + tuning_result_output.uri, + '", "--kms_key_name=', + encryption_spec_key_name, + '", "--gcp_resources_path=', + gcp_resources, + '", "--execution_metrics_path=', + execution_metrics, + ( + '", "--use_custom_job=true", "--use_json=true",' + ' "--log_level=ERROR",' + ' "--executor_input={{$.json_escape[1]}}"]}}]}}' + ), + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/deprecated/default_pipeline.json b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/deprecated/default_pipeline.json new file mode 100644 index 0000000000..5133d9cf2e --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/deprecated/default_pipeline.json @@ -0,0 +1,7974 @@ +{ + "pipelineSpec": { + "components": { + "comp-automl-tabular-cv-trainer": { + "executorLabel": "exec-automl-tabular-cv-trainer", + "inputDefinitions": { + "artifacts": { + "materialized_cv_splits": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "transform_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "tuning_result_input": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "deadline_hours": { + "type": "DOUBLE" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "num_parallel_trials": { + "type": "INT" + }, + "num_selected_trials": { + "type": "INT" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + }, + "single_run_max_secs": { + "type": "INT" + }, + "worker_pool_specs_override": { + "type": "STRING" + }, + "worker_pool_specs_override_json": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "tuning_result_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-automl-tabular-ensemble": { + "executorLabel": "exec-automl-tabular-ensemble", + "inputDefinitions": { + "artifacts": { + "dataset_schema": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "instance_baseline": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "transform_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "tuning_result_input": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "warmup_data": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "encryption_spec_key_name": { + "type": "STRING" + }, + "export_additional_model_without_custom_ops": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "model": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "model_architecture": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "model_without_custom_ops": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-automl-tabular-ensemble-2": { + "executorLabel": "exec-automl-tabular-ensemble-2", + "inputDefinitions": { + "artifacts": { + "dataset_schema": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "instance_baseline": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "transform_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "tuning_result_input": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "warmup_data": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "encryption_spec_key_name": { + "type": "STRING" + }, + "export_additional_model_without_custom_ops": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "model": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "model_architecture": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "model_without_custom_ops": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-automl-tabular-finalizer": { + "executorLabel": "exec-automl-tabular-finalizer", + "inputDefinitions": { + "parameters": { + "encryption_spec_key_name": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-automl-tabular-infra-validator": { + "executorLabel": "exec-automl-tabular-infra-validator", + "inputDefinitions": { + "artifacts": { + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-automl-tabular-infra-validator-2": { + "executorLabel": "exec-automl-tabular-infra-validator-2", + "inputDefinitions": { + "artifacts": { + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-automl-tabular-stage-1-tuner": { + "executorLabel": "exec-automl-tabular-stage-1-tuner", + "inputDefinitions": { + "artifacts": { + "materialized_eval_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "materialized_train_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "transform_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "deadline_hours": { + "type": "DOUBLE" + }, + "disable_early_stopping": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "num_parallel_trials": { + "type": "INT" + }, + "num_selected_trials": { + "type": "INT" + }, + "project": { + "type": "STRING" + }, + "reduce_search_space_mode": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + }, + "run_distillation": { + "type": "STRING" + }, + "single_run_max_secs": { + "type": "INT" + }, + "study_spec_override": { + "type": "STRING" + }, + "study_spec_parameters_override": { + "type": "STRING" + }, + "study_spec_parameters_override_json": { + "type": "STRING" + }, + "tune_feature_selection_rate": { + "type": "STRING" + }, + "worker_pool_specs_override": { + "type": "STRING" + }, + "worker_pool_specs_override_json": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "tuning_result_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-automl-tabular-stage-1-tuner-2": { + "executorLabel": "exec-automl-tabular-stage-1-tuner-2", + "inputDefinitions": { + "artifacts": { + "materialized_eval_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "materialized_train_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "transform_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "deadline_hours": { + "type": "DOUBLE" + }, + "disable_early_stopping": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "num_parallel_trials": { + "type": "INT" + }, + "num_selected_trials": { + "type": "INT" + }, + "project": { + "type": "STRING" + }, + "reduce_search_space_mode": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + }, + "run_distillation": { + "type": "STRING" + }, + "single_run_max_secs": { + "type": "INT" + }, + "study_spec_override": { + "type": "STRING" + }, + "study_spec_parameters_override": { + "type": "STRING" + }, + "study_spec_parameters_override_json": { + "type": "STRING" + }, + "tune_feature_selection_rate": { + "type": "STRING" + }, + "worker_pool_specs_override": { + "type": "STRING" + }, + "worker_pool_specs_override_json": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "tuning_result_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-automl-tabular-transform": { + "executorLabel": "exec-automl-tabular-transform", + "inputDefinitions": { + "artifacts": { + "dataset_schema": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "eval_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "test_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "train_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "dataflow_disk_size_gb": { + "type": "INT" + }, + "dataflow_machine_type": { + "type": "STRING" + }, + "dataflow_max_num_workers": { + "type": "INT" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "materialized_eval_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "materialized_test_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "materialized_train_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "training_schema_uri": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "transform_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-automl-tabular-transform-2": { + "executorLabel": "exec-automl-tabular-transform-2", + "inputDefinitions": { + "artifacts": { + "dataset_schema": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "eval_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "test_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "train_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "dataflow_disk_size_gb": { + "type": "INT" + }, + "dataflow_machine_type": { + "type": "STRING" + }, + "dataflow_max_num_workers": { + "type": "INT" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "materialized_eval_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "materialized_test_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "materialized_train_split": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "training_schema_uri": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "transform_output": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-bool-identity": { + "executorLabel": "exec-bool-identity", + "inputDefinitions": { + "parameters": { + "value": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "parameters": { + "Output": { + "type": "STRING" + } + } + } + }, + "comp-bool-identity-2": { + "executorLabel": "exec-bool-identity-2", + "inputDefinitions": { + "parameters": { + "value": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "parameters": { + "Output": { + "type": "STRING" + } + } + } + }, + "comp-condition-is-distill-4": { + "dag": { + "outputs": { + "artifacts": { + "model-evaluation-3-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-3-evaluation_metrics", + "producerSubtask": "condition-is-evaluation-5" + } + ] + }, + "model-evaluation-4-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-4-evaluation_metrics", + "producerSubtask": "condition-is-evaluation-5" + } + ] + } + } + }, + "tasks": { + "automl-tabular-ensemble-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-ensemble-2" + }, + "dependentTasks": [ + "automl-tabular-stage-1-tuner-2", + "automl-tabular-transform-2" + ], + "inputs": { + "artifacts": { + "dataset_schema": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-dataset_schema" + }, + "instance_baseline": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-instance_baseline" + }, + "metadata": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-metadata" + }, + "transform_output": { + "taskOutputArtifact": { + "outputArtifactKey": "transform_output", + "producerTask": "automl-tabular-transform-2" + } + }, + "tuning_result_input": { + "taskOutputArtifact": { + "outputArtifactKey": "tuning_result_output", + "producerTask": "automl-tabular-stage-1-tuner-2" + } + }, + "warmup_data": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-eval_split" + } + }, + "parameters": { + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "export_additional_model_without_custom_ops": { + "componentInputParameter": "pipelineparam--export_additional_model_without_custom_ops" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "automl-tabular-ensemble-2" + } + }, + "automl-tabular-infra-validator-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-infra-validator-2" + }, + "dependentTasks": [ + "automl-tabular-ensemble-2" + ], + "inputs": { + "artifacts": { + "unmanaged_container_model": { + "taskOutputArtifact": { + "outputArtifactKey": "unmanaged_container_model", + "producerTask": "automl-tabular-ensemble-2" + } + } + } + }, + "taskInfo": { + "name": "automl-tabular-infra-validator-2" + } + }, + "automl-tabular-stage-1-tuner-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-stage-1-tuner-2" + }, + "dependentTasks": [ + "automl-tabular-transform-2" + ], + "inputs": { + "artifacts": { + "materialized_eval_split": { + "taskOutputArtifact": { + "outputArtifactKey": "materialized_eval_split", + "producerTask": "automl-tabular-transform-2" + } + }, + "materialized_train_split": { + "taskOutputArtifact": { + "outputArtifactKey": "materialized_train_split", + "producerTask": "automl-tabular-transform-2" + } + }, + "metadata": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-metadata" + }, + "transform_output": { + "taskOutputArtifact": { + "outputArtifactKey": "transform_output", + "producerTask": "automl-tabular-transform-2" + } + } + }, + "parameters": { + "deadline_hours": { + "componentInputParameter": "pipelineparam--distill_stage_1_deadline_hours" + }, + "disable_early_stopping": { + "componentInputParameter": "pipelineparam--disable_early_stopping" + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "num_parallel_trials": { + "componentInputParameter": "pipelineparam--stage_1_num_parallel_trials" + }, + "num_selected_trials": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "reduce_search_space_mode": { + "componentInputParameter": "pipelineparam--reduce_search_space_mode" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "run_distillation": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "single_run_max_secs": { + "componentInputParameter": "pipelineparam--stage_1_single_run_max_secs" + }, + "study_spec_override": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "study_spec_parameters_override": { + "runtimeValue": { + "constantValue": { + "stringValue": "[]" + } + } + }, + "study_spec_parameters_override_json": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "tune_feature_selection_rate": { + "runtimeValue": { + "constantValue": { + "stringValue": "false" + } + } + }, + "worker_pool_specs_override": { + "componentInputParameter": "pipelineparam--stage_1_tuner_worker_pool_specs_override" + }, + "worker_pool_specs_override_json": { + "runtimeValue": { + "constantValue": { + "stringValue": "[]" + } + } + } + } + }, + "taskInfo": { + "name": "automl-tabular-stage-1-tuner-2" + } + }, + "automl-tabular-transform-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-transform-2" + }, + "dependentTasks": [ + "write-bp-result-path", + "write-bp-result-path-2" + ], + "inputs": { + "artifacts": { + "dataset_schema": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-dataset_schema" + }, + "eval_split": { + "taskOutputArtifact": { + "outputArtifactKey": "result", + "producerTask": "write-bp-result-path-2" + } + }, + "metadata": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-metadata" + }, + "test_split": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-test_split" + }, + "train_split": { + "taskOutputArtifact": { + "outputArtifactKey": "result", + "producerTask": "write-bp-result-path" + } + } + }, + "parameters": { + "dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--transform_dataflow_disk_size_gb" + }, + "dataflow_machine_type": { + "componentInputParameter": "pipelineparam--transform_dataflow_machine_type" + }, + "dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--transform_dataflow_max_num_workers" + }, + "dataflow_service_account": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_subnetwork": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_use_public_ips": { + "runtimeValue": { + "constantValue": { + "stringValue": "true" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "automl-tabular-transform-2" + } + }, + "condition-is-evaluation-5": { + "componentRef": { + "name": "comp-condition-is-evaluation-5" + }, + "dependentTasks": [ + "automl-tabular-ensemble-2", + "model-upload-3" + ], + "inputs": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-2-explanation_metadata_artifact": { + "taskOutputArtifact": { + "outputArtifactKey": "explanation_metadata_artifact", + "producerTask": "automl-tabular-ensemble-2" + } + }, + "pipelineparam--automl-tabular-ensemble-2-unmanaged_container_model": { + "taskOutputArtifact": { + "outputArtifactKey": "unmanaged_container_model", + "producerTask": "automl-tabular-ensemble-2" + } + }, + "pipelineparam--model-upload-3-model": { + "taskOutputArtifact": { + "outputArtifactKey": "model", + "producerTask": "model-upload-3" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-2-explanation_parameters": { + "taskOutputParameter": { + "outputParameterKey": "explanation_parameters", + "producerTask": "automl-tabular-ensemble-2" + } + }, + "pipelineparam--bool-identity-2-Output": { + "componentInputParameter": "pipelineparam--bool-identity-2-Output" + }, + "pipelineparam--bool-identity-Output": { + "componentInputParameter": "pipelineparam--bool-identity-Output" + }, + "pipelineparam--dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "pipelineparam--dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "pipelineparam--dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "pipelineparam--encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "pipelineparam--location": { + "componentInputParameter": "pipelineparam--location" + }, + "pipelineparam--prediction_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "pipelineparam--project": { + "componentInputParameter": "pipelineparam--project" + }, + "pipelineparam--root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json" + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-test_split_json" + }, + "pipelineparam--target_column_name": { + "componentInputParameter": "pipelineparam--target_column_name" + } + } + }, + "taskInfo": { + "name": "condition-is-evaluation-5" + }, + "triggerPolicy": { + "condition": "inputs.parameters['pipelineparam--bool-identity-Output'].string_value == 'true'" + } + }, + "model-batch-predict-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-batch-predict-2" + }, + "dependentTasks": [ + "model-upload-2", + "read-input-uri" + ], + "inputs": { + "artifacts": { + "model": { + "taskOutputArtifact": { + "outputArtifactKey": "model", + "producerTask": "model-upload-2" + } + } + }, + "parameters": { + "accelerator_count": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "accelerator_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_destination_output_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_source_input_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "gcs_destination_output_uri_prefix": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "gcs_source_uris": { + "taskOutputParameter": { + "outputParameterKey": "Output", + "producerTask": "read-input-uri" + } + }, + "generate_explanation": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "instances_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "job_display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "batch-predict-train-split" + } + } + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "machine_type": { + "componentInputParameter": "pipelineparam--distill_batch_predict_machine_type" + }, + "manual_batch_tuning_parameters_batch_size": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "max_replica_count": { + "componentInputParameter": "pipelineparam--distill_batch_predict_max_replica_count" + }, + "model_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "starting_replica_count": { + "componentInputParameter": "pipelineparam--distill_batch_predict_starting_replica_count" + } + } + }, + "taskInfo": { + "name": "model-batch-predict-2" + } + }, + "model-batch-predict-3": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-batch-predict-3" + }, + "dependentTasks": [ + "model-upload-2", + "read-input-uri-2" + ], + "inputs": { + "artifacts": { + "model": { + "taskOutputArtifact": { + "outputArtifactKey": "model", + "producerTask": "model-upload-2" + } + } + }, + "parameters": { + "accelerator_count": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "accelerator_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_destination_output_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_source_input_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "gcs_destination_output_uri_prefix": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "gcs_source_uris": { + "taskOutputParameter": { + "outputParameterKey": "Output", + "producerTask": "read-input-uri-2" + } + }, + "generate_explanation": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "instances_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "job_display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "batch-predict-eval-split" + } + } + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "machine_type": { + "componentInputParameter": "pipelineparam--distill_batch_predict_machine_type" + }, + "manual_batch_tuning_parameters_batch_size": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "max_replica_count": { + "componentInputParameter": "pipelineparam--distill_batch_predict_max_replica_count" + }, + "model_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "starting_replica_count": { + "componentInputParameter": "pipelineparam--distill_batch_predict_starting_replica_count" + } + } + }, + "taskInfo": { + "name": "model-batch-predict-3" + } + }, + "model-upload-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-upload-2" + }, + "dependentTasks": [ + "set-model-can-skip-validation" + ], + "inputs": { + "artifacts": { + "explanation_metadata_artifact": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact" + }, + "unmanaged_container_model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-unmanaged_container_model" + } + }, + "parameters": { + "description": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "automl-tabular-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "componentInputParameter": "pipelineparam--automl-tabular-ensemble-explanation_parameters" + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + } + } + }, + "taskInfo": { + "name": "model-upload-2" + } + }, + "model-upload-3": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-upload-3" + }, + "dependentTasks": [ + "automl-tabular-ensemble-2", + "automl-tabular-infra-validator-2" + ], + "inputs": { + "artifacts": { + "explanation_metadata_artifact": { + "taskOutputArtifact": { + "outputArtifactKey": "explanation_metadata_artifact", + "producerTask": "automl-tabular-ensemble-2" + } + }, + "unmanaged_container_model": { + "taskOutputArtifact": { + "outputArtifactKey": "unmanaged_container_model", + "producerTask": "automl-tabular-ensemble-2" + } + } + }, + "parameters": { + "description": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "automl-tabular-distill-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "taskOutputParameter": { + "outputParameterKey": "explanation_parameters", + "producerTask": "automl-tabular-ensemble-2" + } + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + } + } + }, + "taskInfo": { + "name": "model-upload-3" + } + }, + "read-input-uri": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-read-input-uri" + }, + "inputs": { + "artifacts": { + "split_uri": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-train_split" + } + } + }, + "taskInfo": { + "name": "read-input-uri" + } + }, + "read-input-uri-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-read-input-uri-2" + }, + "inputs": { + "artifacts": { + "split_uri": { + "componentInputArtifact": "pipelineparam--tabular-stats-and-example-gen-eval_split" + } + } + }, + "taskInfo": { + "name": "read-input-uri-2" + } + }, + "set-model-can-skip-validation": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-set-model-can-skip-validation" + }, + "inputs": { + "artifacts": { + "model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-unmanaged_container_model" + } + } + }, + "taskInfo": { + "name": "set-model-can-skip-validation" + } + }, + "write-bp-result-path": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-write-bp-result-path" + }, + "dependentTasks": [ + "model-batch-predict-2" + ], + "inputs": { + "artifacts": { + "bp_job": { + "taskOutputArtifact": { + "outputArtifactKey": "batchpredictionjob", + "producerTask": "model-batch-predict-2" + } + } + } + }, + "taskInfo": { + "name": "write-bp-result-path" + } + }, + "write-bp-result-path-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-write-bp-result-path-2" + }, + "dependentTasks": [ + "model-batch-predict-3" + ], + "inputs": { + "artifacts": { + "bp_job": { + "taskOutputArtifact": { + "outputArtifactKey": "batchpredictionjob", + "producerTask": "model-batch-predict-3" + } + } + } + }, + "taskInfo": { + "name": "write-bp-result-path-2" + } + } + } + }, + "inputDefinitions": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--automl-tabular-ensemble-unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--tabular-stats-and-example-gen-dataset_schema": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--tabular-stats-and-example-gen-eval_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--tabular-stats-and-example-gen-instance_baseline": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--tabular-stats-and-example-gen-metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--tabular-stats-and-example-gen-test_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--tabular-stats-and-example-gen-train_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-explanation_parameters": { + "type": "STRING" + }, + "pipelineparam--bool-identity-2-Output": { + "type": "STRING" + }, + "pipelineparam--bool-identity-Output": { + "type": "STRING" + }, + "pipelineparam--dataflow_service_account": { + "type": "STRING" + }, + "pipelineparam--dataflow_subnetwork": { + "type": "STRING" + }, + "pipelineparam--dataflow_use_public_ips": { + "type": "STRING" + }, + "pipelineparam--disable_early_stopping": { + "type": "STRING" + }, + "pipelineparam--distill_batch_predict_machine_type": { + "type": "STRING" + }, + "pipelineparam--distill_batch_predict_max_replica_count": { + "type": "INT" + }, + "pipelineparam--distill_batch_predict_starting_replica_count": { + "type": "INT" + }, + "pipelineparam--distill_stage_1_deadline_hours": { + "type": "DOUBLE" + }, + "pipelineparam--encryption_spec_key_name": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "type": "INT" + }, + "pipelineparam--export_additional_model_without_custom_ops": { + "type": "STRING" + }, + "pipelineparam--location": { + "type": "STRING" + }, + "pipelineparam--prediction_type": { + "type": "STRING" + }, + "pipelineparam--project": { + "type": "STRING" + }, + "pipelineparam--reduce_search_space_mode": { + "type": "STRING" + }, + "pipelineparam--root_dir": { + "type": "STRING" + }, + "pipelineparam--stage_1_num_parallel_trials": { + "type": "INT" + }, + "pipelineparam--stage_1_single_run_max_secs": { + "type": "INT" + }, + "pipelineparam--stage_1_tuner_worker_pool_specs_override": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "type": "STRING" + }, + "pipelineparam--target_column_name": { + "type": "STRING" + }, + "pipelineparam--transform_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--transform_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--transform_dataflow_max_num_workers": { + "type": "INT" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model-evaluation-3-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-4-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-condition-is-evaluation-3": { + "dag": { + "outputs": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "evaluation_metrics", + "producerSubtask": "model-evaluation-2" + } + ] + }, + "model-evaluation-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "evaluation_metrics", + "producerSubtask": "model-evaluation" + } + ] + } + } + }, + "tasks": { + "model-batch-explanation": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-batch-explanation" + }, + "inputs": { + "artifacts": { + "explanation_metadata_artifact": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact" + }, + "unmanaged_container_model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-unmanaged_container_model" + } + }, + "parameters": { + "accelerator_count": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "accelerator_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_destination_output_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_source_input_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "componentInputParameter": "pipelineparam--automl-tabular-ensemble-explanation_parameters" + }, + "gcs_destination_output_uri_prefix": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "gcs_source_uris": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json" + }, + "generate_explanation": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "instances_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "job_display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" + } + } + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "manual_batch_tuning_parameters_batch_size": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "model_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + } + } + }, + "taskInfo": { + "name": "model-batch-explanation" + } + }, + "model-batch-predict": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-batch-predict" + }, + "inputs": { + "artifacts": { + "unmanaged_container_model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-unmanaged_container_model" + } + }, + "parameters": { + "accelerator_count": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "accelerator_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_destination_output_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_source_input_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "gcs_destination_output_uri_prefix": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "gcs_source_uris": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-test_split_json" + }, + "generate_explanation": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "instances_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "job_display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" + } + } + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "manual_batch_tuning_parameters_batch_size": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "model_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + } + } + }, + "taskInfo": { + "name": "model-batch-predict" + } + }, + "model-evaluation": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-evaluation" + }, + "dependentTasks": [ + "model-batch-predict" + ], + "inputs": { + "artifacts": { + "batch_prediction_job": { + "taskOutputArtifact": { + "outputArtifactKey": "batchpredictionjob", + "producerTask": "model-batch-predict" + } + } + }, + "parameters": { + "class_names": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "classification_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_disk_size": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "dataflow_max_workers_num": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "dataflow_workers_num": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "example_weight_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "generate_feature_attribution": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "ground_truth_column": { + "componentInputParameter": "pipelineparam--target_column_name" + }, + "ground_truth_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "ground_truth_gcs_source": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "key_columns": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "positive_classes": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "prediction_id_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_label_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_score_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "problem_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "model-evaluation" + } + }, + "model-evaluation-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-evaluation-2" + }, + "dependentTasks": [ + "model-batch-explanation" + ], + "inputs": { + "artifacts": { + "batch_prediction_job": { + "taskOutputArtifact": { + "outputArtifactKey": "batchpredictionjob", + "producerTask": "model-batch-explanation" + } + } + }, + "parameters": { + "class_names": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "classification_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_disk_size": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "dataflow_max_workers_num": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "dataflow_workers_num": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "example_weight_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "generate_feature_attribution": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "ground_truth_column": { + "componentInputParameter": "pipelineparam--target_column_name" + }, + "ground_truth_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "ground_truth_gcs_source": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "key_columns": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "positive_classes": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "prediction_id_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_label_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_score_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "problem_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "model-evaluation-2" + } + }, + "model-evaluation-import": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-evaluation-import" + }, + "dependentTasks": [ + "model-evaluation", + "model-evaluation-2" + ], + "inputs": { + "artifacts": { + "explanation": { + "taskOutputArtifact": { + "outputArtifactKey": "evaluation_metrics", + "producerTask": "model-evaluation-2" + } + }, + "metrics": { + "taskOutputArtifact": { + "outputArtifactKey": "evaluation_metrics", + "producerTask": "model-evaluation" + } + }, + "model": { + "componentInputArtifact": "pipelineparam--model-upload-model" + } + }, + "parameters": { + "dataset_path": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataset_paths": { + "runtimeValue": { + "constantValue": { + "stringValue": "[]" + } + } + }, + "dataset_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "problem_type": { + "componentInputParameter": "pipelineparam--prediction_type" + } + } + }, + "taskInfo": { + "name": "model-evaluation-import" + } + } + } + }, + "inputDefinitions": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--automl-tabular-ensemble-unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--model-upload-model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-explanation_parameters": { + "type": "STRING" + }, + "pipelineparam--bool-identity-2-Output": { + "type": "STRING" + }, + "pipelineparam--bool-identity-Output": { + "type": "STRING" + }, + "pipelineparam--dataflow_service_account": { + "type": "STRING" + }, + "pipelineparam--dataflow_subnetwork": { + "type": "STRING" + }, + "pipelineparam--dataflow_use_public_ips": { + "type": "STRING" + }, + "pipelineparam--encryption_spec_key_name": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "type": "INT" + }, + "pipelineparam--location": { + "type": "STRING" + }, + "pipelineparam--prediction_type": { + "type": "STRING" + }, + "pipelineparam--project": { + "type": "STRING" + }, + "pipelineparam--root_dir": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "type": "STRING" + }, + "pipelineparam--target_column_name": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-condition-is-evaluation-5": { + "dag": { + "outputs": { + "artifacts": { + "model-evaluation-3-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "evaluation_metrics", + "producerSubtask": "model-evaluation-3" + } + ] + }, + "model-evaluation-4-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "evaluation_metrics", + "producerSubtask": "model-evaluation-4" + } + ] + } + } + }, + "tasks": { + "model-batch-explanation-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-batch-explanation-2" + }, + "inputs": { + "artifacts": { + "explanation_metadata_artifact": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-2-explanation_metadata_artifact" + }, + "unmanaged_container_model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-2-unmanaged_container_model" + } + }, + "parameters": { + "accelerator_count": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "accelerator_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_destination_output_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_source_input_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "componentInputParameter": "pipelineparam--automl-tabular-ensemble-2-explanation_parameters" + }, + "gcs_destination_output_uri_prefix": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "gcs_source_uris": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json" + }, + "generate_explanation": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "instances_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "job_display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" + } + } + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "manual_batch_tuning_parameters_batch_size": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "model_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + } + } + }, + "taskInfo": { + "name": "model-batch-explanation-2" + } + }, + "model-batch-predict-4": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-batch-predict-4" + }, + "inputs": { + "artifacts": { + "unmanaged_container_model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-2-unmanaged_container_model" + } + }, + "parameters": { + "accelerator_count": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "accelerator_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_destination_output_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "bigquery_source_input_uri": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "gcs_destination_output_uri_prefix": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "gcs_source_uris": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-test_split_json" + }, + "generate_explanation": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "instances_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "tf-record" + } + } + }, + "job_display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" + } + } + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "manual_batch_tuning_parameters_batch_size": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "model_parameters": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + } + } + }, + "taskInfo": { + "name": "model-batch-predict-4" + } + }, + "model-evaluation-3": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-evaluation-3" + }, + "dependentTasks": [ + "model-batch-predict-4" + ], + "inputs": { + "artifacts": { + "batch_prediction_job": { + "taskOutputArtifact": { + "outputArtifactKey": "batchpredictionjob", + "producerTask": "model-batch-predict-4" + } + } + }, + "parameters": { + "class_names": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "classification_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_disk_size": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "dataflow_max_workers_num": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "dataflow_workers_num": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "example_weight_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "generate_feature_attribution": { + "runtimeValue": { + "constantValue": { + "intValue": "0" + } + } + }, + "ground_truth_column": { + "componentInputParameter": "pipelineparam--target_column_name" + }, + "ground_truth_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "ground_truth_gcs_source": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "key_columns": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "positive_classes": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "prediction_id_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_label_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_score_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "problem_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "model-evaluation-3" + } + }, + "model-evaluation-4": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-evaluation-4" + }, + "dependentTasks": [ + "model-batch-explanation-2" + ], + "inputs": { + "artifacts": { + "batch_prediction_job": { + "taskOutputArtifact": { + "outputArtifactKey": "batchpredictionjob", + "producerTask": "model-batch-explanation-2" + } + } + }, + "parameters": { + "class_names": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "classification_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_disk_size": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "dataflow_max_workers_num": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "dataflow_workers_num": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "example_weight_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "generate_feature_attribution": { + "runtimeValue": { + "constantValue": { + "intValue": "1" + } + } + }, + "ground_truth_column": { + "componentInputParameter": "pipelineparam--target_column_name" + }, + "ground_truth_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "ground_truth_gcs_source": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "key_columns": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "positive_classes": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "prediction_id_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_label_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_score_column": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "predictions_format": { + "runtimeValue": { + "constantValue": { + "stringValue": "jsonl" + } + } + }, + "problem_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "model-evaluation-4" + } + }, + "model-evaluation-import-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-evaluation-import-2" + }, + "dependentTasks": [ + "model-evaluation-3", + "model-evaluation-4" + ], + "inputs": { + "artifacts": { + "explanation": { + "taskOutputArtifact": { + "outputArtifactKey": "evaluation_metrics", + "producerTask": "model-evaluation-4" + } + }, + "metrics": { + "taskOutputArtifact": { + "outputArtifactKey": "evaluation_metrics", + "producerTask": "model-evaluation-3" + } + }, + "model": { + "componentInputArtifact": "pipelineparam--model-upload-3-model" + } + }, + "parameters": { + "dataset_path": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataset_paths": { + "runtimeValue": { + "constantValue": { + "stringValue": "[]" + } + } + }, + "dataset_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "problem_type": { + "componentInputParameter": "pipelineparam--prediction_type" + } + } + }, + "taskInfo": { + "name": "model-evaluation-import-2" + } + } + } + }, + "inputDefinitions": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-2-explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--automl-tabular-ensemble-2-unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--model-upload-3-model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-2-explanation_parameters": { + "type": "STRING" + }, + "pipelineparam--bool-identity-2-Output": { + "type": "STRING" + }, + "pipelineparam--bool-identity-Output": { + "type": "STRING" + }, + "pipelineparam--dataflow_service_account": { + "type": "STRING" + }, + "pipelineparam--dataflow_subnetwork": { + "type": "STRING" + }, + "pipelineparam--dataflow_use_public_ips": { + "type": "STRING" + }, + "pipelineparam--encryption_spec_key_name": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "type": "INT" + }, + "pipelineparam--location": { + "type": "STRING" + }, + "pipelineparam--prediction_type": { + "type": "STRING" + }, + "pipelineparam--project": { + "type": "STRING" + }, + "pipelineparam--root_dir": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "type": "STRING" + }, + "pipelineparam--target_column_name": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model-evaluation-3-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-4-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-condition-no-distill-2": { + "dag": { + "outputs": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-2-evaluation_metrics", + "producerSubtask": "condition-is-evaluation-3" + } + ] + }, + "model-evaluation-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-evaluation_metrics", + "producerSubtask": "condition-is-evaluation-3" + } + ] + } + } + }, + "tasks": { + "condition-is-evaluation-3": { + "componentRef": { + "name": "comp-condition-is-evaluation-3" + }, + "dependentTasks": [ + "model-upload" + ], + "inputs": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact" + }, + "pipelineparam--automl-tabular-ensemble-unmanaged_container_model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-unmanaged_container_model" + }, + "pipelineparam--model-upload-model": { + "taskOutputArtifact": { + "outputArtifactKey": "model", + "producerTask": "model-upload" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-explanation_parameters": { + "componentInputParameter": "pipelineparam--automl-tabular-ensemble-explanation_parameters" + }, + "pipelineparam--bool-identity-2-Output": { + "componentInputParameter": "pipelineparam--bool-identity-2-Output" + }, + "pipelineparam--bool-identity-Output": { + "componentInputParameter": "pipelineparam--bool-identity-Output" + }, + "pipelineparam--dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "pipelineparam--dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "pipelineparam--dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "pipelineparam--encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "pipelineparam--location": { + "componentInputParameter": "pipelineparam--location" + }, + "pipelineparam--prediction_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "pipelineparam--project": { + "componentInputParameter": "pipelineparam--project" + }, + "pipelineparam--root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json" + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "componentInputParameter": "pipelineparam--tabular-stats-and-example-gen-test_split_json" + }, + "pipelineparam--target_column_name": { + "componentInputParameter": "pipelineparam--target_column_name" + } + } + }, + "taskInfo": { + "name": "condition-is-evaluation-3" + }, + "triggerPolicy": { + "condition": "inputs.parameters['pipelineparam--bool-identity-Output'].string_value == 'true'" + } + }, + "model-upload": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-model-upload" + }, + "inputs": { + "artifacts": { + "explanation_metadata_artifact": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact" + }, + "unmanaged_container_model": { + "componentInputArtifact": "pipelineparam--automl-tabular-ensemble-unmanaged_container_model" + } + }, + "parameters": { + "description": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "display_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "automl-tabular-model-upload-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" + } + } + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "explanation_metadata": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "explanation_parameters": { + "componentInputParameter": "pipelineparam--automl-tabular-ensemble-explanation_parameters" + }, + "labels": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + } + } + }, + "taskInfo": { + "name": "model-upload" + } + } + } + }, + "inputDefinitions": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "pipelineparam--automl-tabular-ensemble-unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-explanation_parameters": { + "type": "STRING" + }, + "pipelineparam--bool-identity-2-Output": { + "type": "STRING" + }, + "pipelineparam--bool-identity-Output": { + "type": "STRING" + }, + "pipelineparam--dataflow_service_account": { + "type": "STRING" + }, + "pipelineparam--dataflow_subnetwork": { + "type": "STRING" + }, + "pipelineparam--dataflow_use_public_ips": { + "type": "STRING" + }, + "pipelineparam--encryption_spec_key_name": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "type": "INT" + }, + "pipelineparam--location": { + "type": "STRING" + }, + "pipelineparam--prediction_type": { + "type": "STRING" + }, + "pipelineparam--project": { + "type": "STRING" + }, + "pipelineparam--root_dir": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "type": "STRING" + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "type": "STRING" + }, + "pipelineparam--target_column_name": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-exit-handler-1": { + "dag": { + "outputs": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-2-evaluation_metrics", + "producerSubtask": "condition-no-distill-2" + } + ] + }, + "model-evaluation-3-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-3-evaluation_metrics", + "producerSubtask": "condition-is-distill-4" + } + ] + }, + "model-evaluation-4-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-4-evaluation_metrics", + "producerSubtask": "condition-is-distill-4" + } + ] + }, + "model-evaluation-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-evaluation_metrics", + "producerSubtask": "condition-no-distill-2" + } + ] + } + } + }, + "tasks": { + "automl-tabular-cv-trainer": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-cv-trainer" + }, + "dependentTasks": [ + "automl-tabular-stage-1-tuner", + "automl-tabular-transform", + "merge-materialized-splits", + "tabular-stats-and-example-gen" + ], + "inputs": { + "artifacts": { + "materialized_cv_splits": { + "taskOutputArtifact": { + "outputArtifactKey": "splits", + "producerTask": "merge-materialized-splits" + } + }, + "metadata": { + "taskOutputArtifact": { + "outputArtifactKey": "metadata", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "transform_output": { + "taskOutputArtifact": { + "outputArtifactKey": "transform_output", + "producerTask": "automl-tabular-transform" + } + }, + "tuning_result_input": { + "taskOutputArtifact": { + "outputArtifactKey": "tuning_result_output", + "producerTask": "automl-tabular-stage-1-tuner" + } + } + }, + "parameters": { + "deadline_hours": { + "componentInputParameter": "pipelineparam--stage_2_deadline_hours" + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "num_parallel_trials": { + "componentInputParameter": "pipelineparam--stage_2_num_parallel_trials" + }, + "num_selected_trials": { + "componentInputParameter": "pipelineparam--stage_2_num_selected_trials" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "single_run_max_secs": { + "componentInputParameter": "pipelineparam--stage_2_single_run_max_secs" + }, + "worker_pool_specs_override": { + "componentInputParameter": "pipelineparam--cv_trainer_worker_pool_specs_override" + }, + "worker_pool_specs_override_json": { + "runtimeValue": { + "constantValue": { + "stringValue": "[]" + } + } + } + } + }, + "taskInfo": { + "name": "automl-tabular-cv-trainer" + } + }, + "automl-tabular-ensemble": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-ensemble" + }, + "dependentTasks": [ + "automl-tabular-cv-trainer", + "automl-tabular-transform", + "tabular-stats-and-example-gen" + ], + "inputs": { + "artifacts": { + "dataset_schema": { + "taskOutputArtifact": { + "outputArtifactKey": "dataset_schema", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "instance_baseline": { + "taskOutputArtifact": { + "outputArtifactKey": "instance_baseline", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "metadata": { + "taskOutputArtifact": { + "outputArtifactKey": "metadata", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "transform_output": { + "taskOutputArtifact": { + "outputArtifactKey": "transform_output", + "producerTask": "automl-tabular-transform" + } + }, + "tuning_result_input": { + "taskOutputArtifact": { + "outputArtifactKey": "tuning_result_output", + "producerTask": "automl-tabular-cv-trainer" + } + }, + "warmup_data": { + "taskOutputArtifact": { + "outputArtifactKey": "eval_split", + "producerTask": "tabular-stats-and-example-gen" + } + } + }, + "parameters": { + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "export_additional_model_without_custom_ops": { + "componentInputParameter": "pipelineparam--export_additional_model_without_custom_ops" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "automl-tabular-ensemble" + } + }, + "automl-tabular-infra-validator": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-infra-validator" + }, + "dependentTasks": [ + "automl-tabular-ensemble" + ], + "inputs": { + "artifacts": { + "unmanaged_container_model": { + "taskOutputArtifact": { + "outputArtifactKey": "unmanaged_container_model", + "producerTask": "automl-tabular-ensemble" + } + } + } + }, + "taskInfo": { + "name": "automl-tabular-infra-validator" + } + }, + "automl-tabular-stage-1-tuner": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-stage-1-tuner" + }, + "dependentTasks": [ + "automl-tabular-transform", + "tabular-stats-and-example-gen" + ], + "inputs": { + "artifacts": { + "materialized_eval_split": { + "taskOutputArtifact": { + "outputArtifactKey": "materialized_eval_split", + "producerTask": "automl-tabular-transform" + } + }, + "materialized_train_split": { + "taskOutputArtifact": { + "outputArtifactKey": "materialized_train_split", + "producerTask": "automl-tabular-transform" + } + }, + "metadata": { + "taskOutputArtifact": { + "outputArtifactKey": "metadata", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "transform_output": { + "taskOutputArtifact": { + "outputArtifactKey": "transform_output", + "producerTask": "automl-tabular-transform" + } + } + }, + "parameters": { + "deadline_hours": { + "componentInputParameter": "pipelineparam--stage_1_deadline_hours" + }, + "disable_early_stopping": { + "componentInputParameter": "pipelineparam--disable_early_stopping" + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "num_parallel_trials": { + "componentInputParameter": "pipelineparam--stage_1_num_parallel_trials" + }, + "num_selected_trials": { + "componentInputParameter": "pipelineparam--stage_1_num_selected_trials" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "reduce_search_space_mode": { + "componentInputParameter": "pipelineparam--reduce_search_space_mode" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "run_distillation": { + "runtimeValue": { + "constantValue": { + "stringValue": "false" + } + } + }, + "single_run_max_secs": { + "componentInputParameter": "pipelineparam--stage_1_single_run_max_secs" + }, + "study_spec_override": { + "componentInputParameter": "pipelineparam--study_spec_override" + }, + "study_spec_parameters_override": { + "runtimeValue": { + "constantValue": { + "stringValue": "[]" + } + } + }, + "study_spec_parameters_override_json": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "tune_feature_selection_rate": { + "runtimeValue": { + "constantValue": { + "stringValue": "false" + } + } + }, + "worker_pool_specs_override": { + "componentInputParameter": "pipelineparam--stage_1_tuner_worker_pool_specs_override" + }, + "worker_pool_specs_override_json": { + "runtimeValue": { + "constantValue": { + "stringValue": "[]" + } + } + } + } + }, + "taskInfo": { + "name": "automl-tabular-stage-1-tuner" + } + }, + "automl-tabular-transform": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-automl-tabular-transform" + }, + "dependentTasks": [ + "tabular-stats-and-example-gen" + ], + "inputs": { + "artifacts": { + "dataset_schema": { + "taskOutputArtifact": { + "outputArtifactKey": "dataset_schema", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "eval_split": { + "taskOutputArtifact": { + "outputArtifactKey": "eval_split", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "metadata": { + "taskOutputArtifact": { + "outputArtifactKey": "metadata", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "test_split": { + "taskOutputArtifact": { + "outputArtifactKey": "test_split", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "train_split": { + "taskOutputArtifact": { + "outputArtifactKey": "train_split", + "producerTask": "tabular-stats-and-example-gen" + } + } + }, + "parameters": { + "dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--transform_dataflow_disk_size_gb" + }, + "dataflow_machine_type": { + "componentInputParameter": "pipelineparam--transform_dataflow_machine_type" + }, + "dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--transform_dataflow_max_num_workers" + }, + "dataflow_service_account": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + } + } + }, + "taskInfo": { + "name": "automl-tabular-transform" + } + }, + "bool-identity": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-bool-identity" + }, + "inputs": { + "parameters": { + "value": { + "componentInputParameter": "pipelineparam--run_evaluation" + } + } + }, + "taskInfo": { + "name": "bool-identity" + } + }, + "bool-identity-2": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-bool-identity-2" + }, + "inputs": { + "parameters": { + "value": { + "componentInputParameter": "pipelineparam--run_distillation" + } + } + }, + "taskInfo": { + "name": "bool-identity-2" + } + }, + "condition-is-distill-4": { + "componentRef": { + "name": "comp-condition-is-distill-4" + }, + "dependentTasks": [ + "automl-tabular-ensemble", + "automl-tabular-infra-validator", + "bool-identity", + "bool-identity-2", + "tabular-stats-and-example-gen" + ], + "inputs": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact": { + "taskOutputArtifact": { + "outputArtifactKey": "explanation_metadata_artifact", + "producerTask": "automl-tabular-ensemble" + } + }, + "pipelineparam--automl-tabular-ensemble-unmanaged_container_model": { + "taskOutputArtifact": { + "outputArtifactKey": "unmanaged_container_model", + "producerTask": "automl-tabular-ensemble" + } + }, + "pipelineparam--tabular-stats-and-example-gen-dataset_schema": { + "taskOutputArtifact": { + "outputArtifactKey": "dataset_schema", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--tabular-stats-and-example-gen-eval_split": { + "taskOutputArtifact": { + "outputArtifactKey": "eval_split", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--tabular-stats-and-example-gen-instance_baseline": { + "taskOutputArtifact": { + "outputArtifactKey": "instance_baseline", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--tabular-stats-and-example-gen-metadata": { + "taskOutputArtifact": { + "outputArtifactKey": "metadata", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--tabular-stats-and-example-gen-test_split": { + "taskOutputArtifact": { + "outputArtifactKey": "test_split", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--tabular-stats-and-example-gen-train_split": { + "taskOutputArtifact": { + "outputArtifactKey": "train_split", + "producerTask": "tabular-stats-and-example-gen" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-explanation_parameters": { + "taskOutputParameter": { + "outputParameterKey": "explanation_parameters", + "producerTask": "automl-tabular-ensemble" + } + }, + "pipelineparam--bool-identity-2-Output": { + "taskOutputParameter": { + "outputParameterKey": "Output", + "producerTask": "bool-identity-2" + } + }, + "pipelineparam--bool-identity-Output": { + "taskOutputParameter": { + "outputParameterKey": "Output", + "producerTask": "bool-identity" + } + }, + "pipelineparam--dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "pipelineparam--dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "pipelineparam--dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "pipelineparam--disable_early_stopping": { + "componentInputParameter": "pipelineparam--disable_early_stopping" + }, + "pipelineparam--distill_batch_predict_machine_type": { + "componentInputParameter": "pipelineparam--distill_batch_predict_machine_type" + }, + "pipelineparam--distill_batch_predict_max_replica_count": { + "componentInputParameter": "pipelineparam--distill_batch_predict_max_replica_count" + }, + "pipelineparam--distill_batch_predict_starting_replica_count": { + "componentInputParameter": "pipelineparam--distill_batch_predict_starting_replica_count" + }, + "pipelineparam--distill_stage_1_deadline_hours": { + "componentInputParameter": "pipelineparam--distill_stage_1_deadline_hours" + }, + "pipelineparam--encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "pipelineparam--export_additional_model_without_custom_ops": { + "componentInputParameter": "pipelineparam--export_additional_model_without_custom_ops" + }, + "pipelineparam--location": { + "componentInputParameter": "pipelineparam--location" + }, + "pipelineparam--prediction_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "pipelineparam--project": { + "componentInputParameter": "pipelineparam--project" + }, + "pipelineparam--reduce_search_space_mode": { + "componentInputParameter": "pipelineparam--reduce_search_space_mode" + }, + "pipelineparam--root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "pipelineparam--stage_1_num_parallel_trials": { + "componentInputParameter": "pipelineparam--stage_1_num_parallel_trials" + }, + "pipelineparam--stage_1_single_run_max_secs": { + "componentInputParameter": "pipelineparam--stage_1_single_run_max_secs" + }, + "pipelineparam--stage_1_tuner_worker_pool_specs_override": { + "componentInputParameter": "pipelineparam--stage_1_tuner_worker_pool_specs_override" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "taskOutputParameter": { + "outputParameterKey": "downsampled_test_split_json", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "taskOutputParameter": { + "outputParameterKey": "test_split_json", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--target_column_name": { + "componentInputParameter": "pipelineparam--target_column_name" + }, + "pipelineparam--transform_dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--transform_dataflow_disk_size_gb" + }, + "pipelineparam--transform_dataflow_machine_type": { + "componentInputParameter": "pipelineparam--transform_dataflow_machine_type" + }, + "pipelineparam--transform_dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--transform_dataflow_max_num_workers" + } + } + }, + "taskInfo": { + "name": "condition-is-distill-4" + }, + "triggerPolicy": { + "condition": "inputs.parameters['pipelineparam--bool-identity-2-Output'].string_value == 'true'" + } + }, + "condition-no-distill-2": { + "componentRef": { + "name": "comp-condition-no-distill-2" + }, + "dependentTasks": [ + "automl-tabular-ensemble", + "automl-tabular-infra-validator", + "bool-identity", + "bool-identity-2", + "tabular-stats-and-example-gen" + ], + "inputs": { + "artifacts": { + "pipelineparam--automl-tabular-ensemble-explanation_metadata_artifact": { + "taskOutputArtifact": { + "outputArtifactKey": "explanation_metadata_artifact", + "producerTask": "automl-tabular-ensemble" + } + }, + "pipelineparam--automl-tabular-ensemble-unmanaged_container_model": { + "taskOutputArtifact": { + "outputArtifactKey": "unmanaged_container_model", + "producerTask": "automl-tabular-ensemble" + } + } + }, + "parameters": { + "pipelineparam--automl-tabular-ensemble-explanation_parameters": { + "taskOutputParameter": { + "outputParameterKey": "explanation_parameters", + "producerTask": "automl-tabular-ensemble" + } + }, + "pipelineparam--bool-identity-2-Output": { + "taskOutputParameter": { + "outputParameterKey": "Output", + "producerTask": "bool-identity-2" + } + }, + "pipelineparam--bool-identity-Output": { + "taskOutputParameter": { + "outputParameterKey": "Output", + "producerTask": "bool-identity" + } + }, + "pipelineparam--dataflow_service_account": { + "componentInputParameter": "pipelineparam--dataflow_service_account" + }, + "pipelineparam--dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "pipelineparam--dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "pipelineparam--encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_machine_type" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_max_replica_count" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "componentInputParameter": "pipelineparam--evaluation_batch_predict_starting_replica_count" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_disk_size_gb" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_machine_type" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--evaluation_dataflow_max_num_workers" + }, + "pipelineparam--location": { + "componentInputParameter": "pipelineparam--location" + }, + "pipelineparam--prediction_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "pipelineparam--project": { + "componentInputParameter": "pipelineparam--project" + }, + "pipelineparam--root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "pipelineparam--tabular-stats-and-example-gen-downsampled_test_split_json": { + "taskOutputParameter": { + "outputParameterKey": "downsampled_test_split_json", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--tabular-stats-and-example-gen-test_split_json": { + "taskOutputParameter": { + "outputParameterKey": "test_split_json", + "producerTask": "tabular-stats-and-example-gen" + } + }, + "pipelineparam--target_column_name": { + "componentInputParameter": "pipelineparam--target_column_name" + } + } + }, + "taskInfo": { + "name": "condition-no-distill-2" + }, + "triggerPolicy": { + "condition": "inputs.parameters['pipelineparam--bool-identity-2-Output'].string_value == 'false'" + } + }, + "merge-materialized-splits": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-merge-materialized-splits" + }, + "dependentTasks": [ + "automl-tabular-transform" + ], + "inputs": { + "artifacts": { + "split_0": { + "taskOutputArtifact": { + "outputArtifactKey": "materialized_train_split", + "producerTask": "automl-tabular-transform" + } + }, + "split_1": { + "taskOutputArtifact": { + "outputArtifactKey": "materialized_eval_split", + "producerTask": "automl-tabular-transform" + } + } + } + }, + "taskInfo": { + "name": "merge-materialized-splits" + } + }, + "tabular-stats-and-example-gen": { + "cachingOptions": { + "enableCache": true + }, + "componentRef": { + "name": "comp-tabular-stats-and-example-gen" + }, + "inputs": { + "parameters": { + "additional_experiments": { + "componentInputParameter": "pipelineparam--additional_experiments" + }, + "additional_experiments_json": { + "runtimeValue": { + "constantValue": { + "stringValue": "{}" + } + } + }, + "data_source": { + "componentInputParameter": "pipelineparam--data_source" + }, + "data_source_bigquery_table_path": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "data_source_csv_filenames": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_disk_size_gb": { + "componentInputParameter": "pipelineparam--stats_and_example_gen_dataflow_disk_size_gb" + }, + "dataflow_machine_type": { + "componentInputParameter": "pipelineparam--stats_and_example_gen_dataflow_machine_type" + }, + "dataflow_max_num_workers": { + "componentInputParameter": "pipelineparam--stats_and_example_gen_dataflow_max_num_workers" + }, + "dataflow_service_account": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "dataflow_subnetwork": { + "componentInputParameter": "pipelineparam--dataflow_subnetwork" + }, + "dataflow_use_public_ips": { + "componentInputParameter": "pipelineparam--dataflow_use_public_ips" + }, + "encryption_spec_key_name": { + "componentInputParameter": "pipelineparam--encryption_spec_key_name" + }, + "location": { + "componentInputParameter": "pipelineparam--location" + }, + "optimization_objective": { + "componentInputParameter": "pipelineparam--optimization_objective" + }, + "optimization_objective_precision_value": { + "componentInputParameter": "pipelineparam--optimization_objective_precision_value" + }, + "optimization_objective_recall_value": { + "componentInputParameter": "pipelineparam--optimization_objective_recall_value" + }, + "predefined_split_key": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "prediction_type": { + "componentInputParameter": "pipelineparam--prediction_type" + }, + "project": { + "componentInputParameter": "pipelineparam--project" + }, + "request_type": { + "runtimeValue": { + "constantValue": { + "stringValue": "COLUMN_STATS_ONLY" + } + } + }, + "root_dir": { + "componentInputParameter": "pipelineparam--root_dir" + }, + "run_distillation": { + "componentInputParameter": "pipelineparam--run_distillation" + }, + "split_spec": { + "componentInputParameter": "pipelineparam--split_spec" + }, + "stratified_split_key": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "target_column_name": { + "componentInputParameter": "pipelineparam--target_column_name" + }, + "test_fraction": { + "runtimeValue": { + "constantValue": { + "stringValue": "-1" + } + } + }, + "timestamp_split_key": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "training_fraction": { + "runtimeValue": { + "constantValue": { + "stringValue": "-1" + } + } + }, + "transformations": { + "componentInputParameter": "pipelineparam--transformations" + }, + "transformations_path": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "validation_fraction": { + "runtimeValue": { + "constantValue": { + "stringValue": "-1" + } + } + }, + "weight_column_name": { + "componentInputParameter": "pipelineparam--weight_column_name" + } + } + }, + "taskInfo": { + "name": "tabular-stats-and-example-gen" + } + } + } + }, + "inputDefinitions": { + "parameters": { + "pipelineparam--additional_experiments": { + "type": "STRING" + }, + "pipelineparam--cv_trainer_worker_pool_specs_override": { + "type": "STRING" + }, + "pipelineparam--data_source": { + "type": "STRING" + }, + "pipelineparam--dataflow_service_account": { + "type": "STRING" + }, + "pipelineparam--dataflow_subnetwork": { + "type": "STRING" + }, + "pipelineparam--dataflow_use_public_ips": { + "type": "STRING" + }, + "pipelineparam--disable_early_stopping": { + "type": "STRING" + }, + "pipelineparam--distill_batch_predict_machine_type": { + "type": "STRING" + }, + "pipelineparam--distill_batch_predict_max_replica_count": { + "type": "INT" + }, + "pipelineparam--distill_batch_predict_starting_replica_count": { + "type": "INT" + }, + "pipelineparam--distill_stage_1_deadline_hours": { + "type": "DOUBLE" + }, + "pipelineparam--encryption_spec_key_name": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "type": "INT" + }, + "pipelineparam--export_additional_model_without_custom_ops": { + "type": "STRING" + }, + "pipelineparam--location": { + "type": "STRING" + }, + "pipelineparam--optimization_objective": { + "type": "STRING" + }, + "pipelineparam--optimization_objective_precision_value": { + "type": "DOUBLE" + }, + "pipelineparam--optimization_objective_recall_value": { + "type": "DOUBLE" + }, + "pipelineparam--prediction_type": { + "type": "STRING" + }, + "pipelineparam--project": { + "type": "STRING" + }, + "pipelineparam--reduce_search_space_mode": { + "type": "STRING" + }, + "pipelineparam--root_dir": { + "type": "STRING" + }, + "pipelineparam--run_distillation": { + "type": "STRING" + }, + "pipelineparam--run_evaluation": { + "type": "STRING" + }, + "pipelineparam--split_spec": { + "type": "STRING" + }, + "pipelineparam--stage_1_deadline_hours": { + "type": "DOUBLE" + }, + "pipelineparam--stage_1_num_parallel_trials": { + "type": "INT" + }, + "pipelineparam--stage_1_num_selected_trials": { + "type": "INT" + }, + "pipelineparam--stage_1_single_run_max_secs": { + "type": "INT" + }, + "pipelineparam--stage_1_tuner_worker_pool_specs_override": { + "type": "STRING" + }, + "pipelineparam--stage_2_deadline_hours": { + "type": "DOUBLE" + }, + "pipelineparam--stage_2_num_parallel_trials": { + "type": "INT" + }, + "pipelineparam--stage_2_num_selected_trials": { + "type": "INT" + }, + "pipelineparam--stage_2_single_run_max_secs": { + "type": "INT" + }, + "pipelineparam--stats_and_example_gen_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--stats_and_example_gen_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--stats_and_example_gen_dataflow_max_num_workers": { + "type": "INT" + }, + "pipelineparam--study_spec_override": { + "type": "STRING" + }, + "pipelineparam--target_column_name": { + "type": "STRING" + }, + "pipelineparam--transform_dataflow_disk_size_gb": { + "type": "INT" + }, + "pipelineparam--transform_dataflow_machine_type": { + "type": "STRING" + }, + "pipelineparam--transform_dataflow_max_num_workers": { + "type": "INT" + }, + "pipelineparam--transformations": { + "type": "STRING" + }, + "pipelineparam--weight_column_name": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-3-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-4-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-merge-materialized-splits": { + "executorLabel": "exec-merge-materialized-splits", + "inputDefinitions": { + "artifacts": { + "split_0": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "split_1": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + } + }, + "outputDefinitions": { + "artifacts": { + "splits": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-model-batch-explanation": { + "executorLabel": "exec-model-batch-explanation", + "inputDefinitions": { + "artifacts": { + "explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "accelerator_count": { + "type": "INT" + }, + "accelerator_type": { + "type": "STRING" + }, + "bigquery_destination_output_uri": { + "type": "STRING" + }, + "bigquery_source_input_uri": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcs_destination_output_uri_prefix": { + "type": "STRING" + }, + "gcs_source_uris": { + "type": "STRING" + }, + "generate_explanation": { + "type": "STRING" + }, + "instances_format": { + "type": "STRING" + }, + "job_display_name": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "machine_type": { + "type": "STRING" + }, + "manual_batch_tuning_parameters_batch_size": { + "type": "INT" + }, + "max_replica_count": { + "type": "INT" + }, + "model_parameters": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "starting_replica_count": { + "type": "INT" + } + } + }, + "outputDefinitions": { + "artifacts": { + "batchpredictionjob": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + }, + "bigquery_output_table": { + "artifactType": { + "schemaTitle": "google.BQTable", + "schemaVersion": "0.0.1" + } + }, + "gcs_output_directory": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-batch-explanation-2": { + "executorLabel": "exec-model-batch-explanation-2", + "inputDefinitions": { + "artifacts": { + "explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "accelerator_count": { + "type": "INT" + }, + "accelerator_type": { + "type": "STRING" + }, + "bigquery_destination_output_uri": { + "type": "STRING" + }, + "bigquery_source_input_uri": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcs_destination_output_uri_prefix": { + "type": "STRING" + }, + "gcs_source_uris": { + "type": "STRING" + }, + "generate_explanation": { + "type": "STRING" + }, + "instances_format": { + "type": "STRING" + }, + "job_display_name": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "machine_type": { + "type": "STRING" + }, + "manual_batch_tuning_parameters_batch_size": { + "type": "INT" + }, + "max_replica_count": { + "type": "INT" + }, + "model_parameters": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "starting_replica_count": { + "type": "INT" + } + } + }, + "outputDefinitions": { + "artifacts": { + "batchpredictionjob": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + }, + "bigquery_output_table": { + "artifactType": { + "schemaTitle": "google.BQTable", + "schemaVersion": "0.0.1" + } + }, + "gcs_output_directory": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-batch-predict": { + "executorLabel": "exec-model-batch-predict", + "inputDefinitions": { + "artifacts": { + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "accelerator_count": { + "type": "INT" + }, + "accelerator_type": { + "type": "STRING" + }, + "bigquery_destination_output_uri": { + "type": "STRING" + }, + "bigquery_source_input_uri": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcs_destination_output_uri_prefix": { + "type": "STRING" + }, + "gcs_source_uris": { + "type": "STRING" + }, + "generate_explanation": { + "type": "STRING" + }, + "instances_format": { + "type": "STRING" + }, + "job_display_name": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "machine_type": { + "type": "STRING" + }, + "manual_batch_tuning_parameters_batch_size": { + "type": "INT" + }, + "max_replica_count": { + "type": "INT" + }, + "model_parameters": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "starting_replica_count": { + "type": "INT" + } + } + }, + "outputDefinitions": { + "artifacts": { + "batchpredictionjob": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + }, + "bigquery_output_table": { + "artifactType": { + "schemaTitle": "google.BQTable", + "schemaVersion": "0.0.1" + } + }, + "gcs_output_directory": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-batch-predict-2": { + "executorLabel": "exec-model-batch-predict-2", + "inputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "accelerator_count": { + "type": "INT" + }, + "accelerator_type": { + "type": "STRING" + }, + "bigquery_destination_output_uri": { + "type": "STRING" + }, + "bigquery_source_input_uri": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcs_destination_output_uri_prefix": { + "type": "STRING" + }, + "gcs_source_uris": { + "type": "STRING" + }, + "generate_explanation": { + "type": "STRING" + }, + "instances_format": { + "type": "STRING" + }, + "job_display_name": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "machine_type": { + "type": "STRING" + }, + "manual_batch_tuning_parameters_batch_size": { + "type": "INT" + }, + "max_replica_count": { + "type": "INT" + }, + "model_parameters": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "starting_replica_count": { + "type": "INT" + } + } + }, + "outputDefinitions": { + "artifacts": { + "batchpredictionjob": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + }, + "bigquery_output_table": { + "artifactType": { + "schemaTitle": "google.BQTable", + "schemaVersion": "0.0.1" + } + }, + "gcs_output_directory": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-batch-predict-3": { + "executorLabel": "exec-model-batch-predict-3", + "inputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "accelerator_count": { + "type": "INT" + }, + "accelerator_type": { + "type": "STRING" + }, + "bigquery_destination_output_uri": { + "type": "STRING" + }, + "bigquery_source_input_uri": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcs_destination_output_uri_prefix": { + "type": "STRING" + }, + "gcs_source_uris": { + "type": "STRING" + }, + "generate_explanation": { + "type": "STRING" + }, + "instances_format": { + "type": "STRING" + }, + "job_display_name": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "machine_type": { + "type": "STRING" + }, + "manual_batch_tuning_parameters_batch_size": { + "type": "INT" + }, + "max_replica_count": { + "type": "INT" + }, + "model_parameters": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "starting_replica_count": { + "type": "INT" + } + } + }, + "outputDefinitions": { + "artifacts": { + "batchpredictionjob": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + }, + "bigquery_output_table": { + "artifactType": { + "schemaTitle": "google.BQTable", + "schemaVersion": "0.0.1" + } + }, + "gcs_output_directory": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-batch-predict-4": { + "executorLabel": "exec-model-batch-predict-4", + "inputDefinitions": { + "artifacts": { + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "accelerator_count": { + "type": "INT" + }, + "accelerator_type": { + "type": "STRING" + }, + "bigquery_destination_output_uri": { + "type": "STRING" + }, + "bigquery_source_input_uri": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "gcs_destination_output_uri_prefix": { + "type": "STRING" + }, + "gcs_source_uris": { + "type": "STRING" + }, + "generate_explanation": { + "type": "STRING" + }, + "instances_format": { + "type": "STRING" + }, + "job_display_name": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "machine_type": { + "type": "STRING" + }, + "manual_batch_tuning_parameters_batch_size": { + "type": "INT" + }, + "max_replica_count": { + "type": "INT" + }, + "model_parameters": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "starting_replica_count": { + "type": "INT" + } + } + }, + "outputDefinitions": { + "artifacts": { + "batchpredictionjob": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + }, + "bigquery_output_table": { + "artifactType": { + "schemaTitle": "google.BQTable", + "schemaVersion": "0.0.1" + } + }, + "gcs_output_directory": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-evaluation": { + "executorLabel": "exec-model-evaluation", + "inputDefinitions": { + "artifacts": { + "batch_prediction_job": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "class_names": { + "type": "STRING" + }, + "classification_type": { + "type": "STRING" + }, + "dataflow_disk_size": { + "type": "INT" + }, + "dataflow_machine_type": { + "type": "STRING" + }, + "dataflow_max_workers_num": { + "type": "INT" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "dataflow_workers_num": { + "type": "INT" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "example_weight_column": { + "type": "STRING" + }, + "generate_feature_attribution": { + "type": "STRING" + }, + "ground_truth_column": { + "type": "STRING" + }, + "ground_truth_format": { + "type": "STRING" + }, + "ground_truth_gcs_source": { + "type": "STRING" + }, + "key_columns": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "positive_classes": { + "type": "STRING" + }, + "prediction_id_column": { + "type": "STRING" + }, + "prediction_label_column": { + "type": "STRING" + }, + "prediction_score_column": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "problem_type": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-evaluation-2": { + "executorLabel": "exec-model-evaluation-2", + "inputDefinitions": { + "artifacts": { + "batch_prediction_job": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "class_names": { + "type": "STRING" + }, + "classification_type": { + "type": "STRING" + }, + "dataflow_disk_size": { + "type": "INT" + }, + "dataflow_machine_type": { + "type": "STRING" + }, + "dataflow_max_workers_num": { + "type": "INT" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "dataflow_workers_num": { + "type": "INT" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "example_weight_column": { + "type": "STRING" + }, + "generate_feature_attribution": { + "type": "STRING" + }, + "ground_truth_column": { + "type": "STRING" + }, + "ground_truth_format": { + "type": "STRING" + }, + "ground_truth_gcs_source": { + "type": "STRING" + }, + "key_columns": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "positive_classes": { + "type": "STRING" + }, + "prediction_id_column": { + "type": "STRING" + }, + "prediction_label_column": { + "type": "STRING" + }, + "prediction_score_column": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "problem_type": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-evaluation-3": { + "executorLabel": "exec-model-evaluation-3", + "inputDefinitions": { + "artifacts": { + "batch_prediction_job": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "class_names": { + "type": "STRING" + }, + "classification_type": { + "type": "STRING" + }, + "dataflow_disk_size": { + "type": "INT" + }, + "dataflow_machine_type": { + "type": "STRING" + }, + "dataflow_max_workers_num": { + "type": "INT" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "dataflow_workers_num": { + "type": "INT" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "example_weight_column": { + "type": "STRING" + }, + "generate_feature_attribution": { + "type": "STRING" + }, + "ground_truth_column": { + "type": "STRING" + }, + "ground_truth_format": { + "type": "STRING" + }, + "ground_truth_gcs_source": { + "type": "STRING" + }, + "key_columns": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "positive_classes": { + "type": "STRING" + }, + "prediction_id_column": { + "type": "STRING" + }, + "prediction_label_column": { + "type": "STRING" + }, + "prediction_score_column": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "problem_type": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-evaluation-4": { + "executorLabel": "exec-model-evaluation-4", + "inputDefinitions": { + "artifacts": { + "batch_prediction_job": { + "artifactType": { + "schemaTitle": "google.VertexBatchPredictionJob", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "class_names": { + "type": "STRING" + }, + "classification_type": { + "type": "STRING" + }, + "dataflow_disk_size": { + "type": "INT" + }, + "dataflow_machine_type": { + "type": "STRING" + }, + "dataflow_max_workers_num": { + "type": "INT" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "dataflow_workers_num": { + "type": "INT" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "example_weight_column": { + "type": "STRING" + }, + "generate_feature_attribution": { + "type": "STRING" + }, + "ground_truth_column": { + "type": "STRING" + }, + "ground_truth_format": { + "type": "STRING" + }, + "ground_truth_gcs_source": { + "type": "STRING" + }, + "key_columns": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "positive_classes": { + "type": "STRING" + }, + "prediction_id_column": { + "type": "STRING" + }, + "prediction_label_column": { + "type": "STRING" + }, + "prediction_score_column": { + "type": "STRING" + }, + "predictions_format": { + "type": "STRING" + }, + "problem_type": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-evaluation-import": { + "executorLabel": "exec-model-evaluation-import", + "inputDefinitions": { + "artifacts": { + "explanation": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "dataset_path": { + "type": "STRING" + }, + "dataset_paths": { + "type": "STRING" + }, + "dataset_type": { + "type": "STRING" + }, + "display_name": { + "type": "STRING" + }, + "problem_type": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-evaluation-import-2": { + "executorLabel": "exec-model-evaluation-import-2", + "inputDefinitions": { + "artifacts": { + "explanation": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "dataset_path": { + "type": "STRING" + }, + "dataset_paths": { + "type": "STRING" + }, + "dataset_type": { + "type": "STRING" + }, + "display_name": { + "type": "STRING" + }, + "problem_type": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-upload": { + "executorLabel": "exec-model-upload", + "inputDefinitions": { + "artifacts": { + "explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "description": { + "type": "STRING" + }, + "display_name": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-upload-2": { + "executorLabel": "exec-model-upload-2", + "inputDefinitions": { + "artifacts": { + "explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "description": { + "type": "STRING" + }, + "display_name": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-model-upload-3": { + "executorLabel": "exec-model-upload-3", + "inputDefinitions": { + "artifacts": { + "explanation_metadata_artifact": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "unmanaged_container_model": { + "artifactType": { + "schemaTitle": "google.UnmanagedContainerModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "description": { + "type": "STRING" + }, + "display_name": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "explanation_metadata": { + "type": "STRING" + }, + "explanation_parameters": { + "type": "STRING" + }, + "labels": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "project": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "google.VertexModel", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "gcp_resources": { + "type": "STRING" + } + } + } + }, + "comp-read-input-uri": { + "executorLabel": "exec-read-input-uri", + "inputDefinitions": { + "artifacts": { + "split_uri": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + } + }, + "outputDefinitions": { + "parameters": { + "Output": { + "type": "STRING" + } + } + } + }, + "comp-read-input-uri-2": { + "executorLabel": "exec-read-input-uri-2", + "inputDefinitions": { + "artifacts": { + "split_uri": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + } + }, + "outputDefinitions": { + "parameters": { + "Output": { + "type": "STRING" + } + } + } + }, + "comp-set-model-can-skip-validation": { + "executorLabel": "exec-set-model-can-skip-validation", + "inputDefinitions": { + "artifacts": { + "model": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-tabular-stats-and-example-gen": { + "executorLabel": "exec-tabular-stats-and-example-gen", + "inputDefinitions": { + "parameters": { + "additional_experiments": { + "type": "STRING" + }, + "additional_experiments_json": { + "type": "STRING" + }, + "data_source": { + "type": "STRING" + }, + "data_source_bigquery_table_path": { + "type": "STRING" + }, + "data_source_csv_filenames": { + "type": "STRING" + }, + "dataflow_disk_size_gb": { + "type": "INT" + }, + "dataflow_machine_type": { + "type": "STRING" + }, + "dataflow_max_num_workers": { + "type": "INT" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "optimization_objective": { + "type": "STRING" + }, + "optimization_objective_precision_value": { + "type": "DOUBLE" + }, + "optimization_objective_recall_value": { + "type": "DOUBLE" + }, + "predefined_split_key": { + "type": "STRING" + }, + "prediction_type": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "request_type": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + }, + "run_distillation": { + "type": "STRING" + }, + "split_spec": { + "type": "STRING" + }, + "stratified_split_key": { + "type": "STRING" + }, + "target_column_name": { + "type": "STRING" + }, + "test_fraction": { + "type": "DOUBLE" + }, + "timestamp_split_key": { + "type": "STRING" + }, + "training_fraction": { + "type": "DOUBLE" + }, + "transformations": { + "type": "STRING" + }, + "transformations_path": { + "type": "STRING" + }, + "validation_fraction": { + "type": "DOUBLE" + }, + "weight_column_name": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "dataset_schema": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "dataset_stats": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "eval_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "instance_baseline": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "metadata": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + }, + "test_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + }, + "train_split": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + }, + "parameters": { + "downsampled_test_split_json": { + "type": "STRING" + }, + "gcp_resources": { + "type": "STRING" + }, + "test_split_json": { + "type": "STRING" + } + } + } + }, + "comp-write-bp-result-path": { + "executorLabel": "exec-write-bp-result-path", + "inputDefinitions": { + "artifacts": { + "bp_job": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + } + }, + "outputDefinitions": { + "artifacts": { + "result": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "comp-write-bp-result-path-2": { + "executorLabel": "exec-write-bp-result-path-2", + "inputDefinitions": { + "artifacts": { + "bp_job": { + "artifactType": { + "schemaTitle": "system.Artifact", + "schemaVersion": "0.0.1" + } + } + } + }, + "outputDefinitions": { + "artifacts": { + "result": { + "artifactType": { + "schemaTitle": "system.Dataset", + "schemaVersion": "0.0.1" + } + } + } + } + } + }, + "deploymentSpec": { + "executors": { + "exec-automl-tabular-cv-trainer": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-cv-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"l2l_cv_tuner\", \"--transform_output_path={{$.inputs.artifacts['transform_output'].uri}}\", \"--training_docker_uri=us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--worker_pool_specs_override={{$.inputs.parameters['worker_pool_specs_override']}}\", \"--num_parallel_trial={{$.inputs.parameters['num_parallel_trials']}}\", \"--single_run_max_secs={{$.inputs.parameters['single_run_max_secs']}}\", \"--deadline_hours={{$.inputs.parameters['deadline_hours']}}\", \"--valid_trials_completed_threshold=0.7\", \"--num_selected_trials={{$.inputs.parameters['num_selected_trials']}}\", \"--lro_job_info={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/lro\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--metadata_path={{$.inputs.artifacts['metadata'].uri}}\", \"--materialized_cv_splits={{$.inputs.artifacts['materialized_cv_splits'].uri}}\", \"--tuning_result_input_path={{$.inputs.artifacts['tuning_result_input'].uri}}\", \"--tuning_result_output_path={{$.outputs.artifacts['tuning_result_output'].uri}}\", \"--kms_key_name={{$.inputs.parameters['encryption_spec_key_name']}}\", \"--use_custom_job=true\", \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-automl-tabular-ensemble": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"ensemble\", \"--transform_output_path={{$.inputs.artifacts['transform_output'].uri}}\", \"--model_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--export_custom_model={{$.inputs.parameters['export_additional_model_without_custom_ops']}}\", \"--metadata_path={{$.inputs.artifacts['metadata'].uri}}\", \"--dataset_schema_path={{$.inputs.artifacts['dataset_schema'].uri}}\", \"--tuning_result_input_path={{$.inputs.artifacts['tuning_result_input'].uri}}\", \"--instance_baseline_path={{$.inputs.artifacts['instance_baseline'].uri}}\", \"--warmup_data={{$.inputs.artifacts['warmup_data'].uri}}\", \"--prediction_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod\", \"--model_path={{$.outputs.artifacts['model'].uri}}\", \"--custom_model_path={{$.outputs.artifacts['model_without_custom_ops'].uri}}\", \"--explanation_metadata_path={{$.outputs.parameters['explanation_metadata'].output_file}},{{$.outputs.artifacts['explanation_metadata_artifact'].uri}}\", \"--explanation_parameters_path={{$.outputs.parameters['explanation_parameters'].output_file}}\", \"--model_architecture_path={{$.outputs.artifacts['model_architecture'].uri}}\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-automl-tabular-ensemble-2": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-ensemble-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-highmem-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"ensemble\", \"--transform_output_path={{$.inputs.artifacts['transform_output'].uri}}\", \"--model_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/model\", \"--custom_model_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/custom_model\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--export_custom_model={{$.inputs.parameters['export_additional_model_without_custom_ops']}}\", \"--metadata_path={{$.inputs.artifacts['metadata'].uri}}\", \"--dataset_schema_path={{$.inputs.artifacts['dataset_schema'].uri}}\", \"--tuning_result_input_path={{$.inputs.artifacts['tuning_result_input'].uri}}\", \"--instance_baseline_path={{$.inputs.artifacts['instance_baseline'].uri}}\", \"--warmup_data={{$.inputs.artifacts['warmup_data'].uri}}\", \"--prediction_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod\", \"--model_path={{$.outputs.artifacts['model'].uri}}\", \"--custom_model_path={{$.outputs.artifacts['model_without_custom_ops'].uri}}\", \"--explanation_metadata_path={{$.outputs.parameters['explanation_metadata'].output_file}},{{$.outputs.artifacts['explanation_metadata_artifact'].uri}}\", \"--explanation_parameters_path={{$.outputs.parameters['explanation_parameters'].output_file}}\", \"--model_architecture_path={{$.outputs.artifacts['model_architecture'].uri}}\", \"--use_json=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-automl-tabular-finalizer": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-finalizer-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"cancel_l2l_tuner\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--cleanup_lro_job_infos={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/lro\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-automl-tabular-infra-validator": { + "container": { + "args": [ + "--executor_input", + "{{$}}" + ], + "image": "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod", + "resources": { + "cpuLimit": 8.0, + "memoryLimit": 52.0 + } + } + }, + "exec-automl-tabular-infra-validator-2": { + "container": { + "args": [ + "--executor_input", + "{{$}}" + ], + "image": "us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod", + "resources": { + "cpuLimit": 8.0, + "memoryLimit": 52.0 + } + } + }, + "exec-automl-tabular-stage-1-tuner": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path={{$.inputs.artifacts['transform_output'].uri}}\", \"--training_docker_uri=us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"--disable_early_stopping={{$.inputs.parameters['disable_early_stopping']}}\", \"--tune_feature_selection_rate={{$.inputs.parameters['tune_feature_selection_rate']}}\", \"--reduce_search_space_mode={{$.inputs.parameters['reduce_search_space_mode']}}\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--study_spec_override={{$.inputs.parameters['study_spec_override']}}\", \"--worker_pool_specs_override={{$.inputs.parameters['worker_pool_specs_override']}}\", \"--num_parallel_trial={{$.inputs.parameters['num_parallel_trials']}}\", \"--single_run_max_secs={{$.inputs.parameters['single_run_max_secs']}}\", \"--deadline_hours={{$.inputs.parameters['deadline_hours']}}\", \"--num_selected_trials={{$.inputs.parameters['num_selected_trials']}}\", \"--lro_job_info={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/lro\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--metadata_path={{$.inputs.artifacts['metadata'].uri}}\", \"--materialized_train_split={{$.inputs.artifacts['materialized_train_split'].uri}}\", \"--materialized_eval_split={{$.inputs.artifacts['materialized_eval_split'].uri}}\", \"--is_distill={{$.inputs.parameters['run_distillation']}}\", \"--tuning_result_output_path={{$.outputs.artifacts['tuning_result_output'].uri}}\", \"--kms_key_name={{$.inputs.parameters['encryption_spec_key_name']}}\", \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-automl-tabular-stage-1-tuner-2": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-stage-1-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"l2l_stage_1_tuner\", \"--transform_output_path={{$.inputs.artifacts['transform_output'].uri}}\", \"--training_docker_uri=us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"--disable_early_stopping={{$.inputs.parameters['disable_early_stopping']}}\", \"--tune_feature_selection_rate={{$.inputs.parameters['tune_feature_selection_rate']}}\", \"--reduce_search_space_mode={{$.inputs.parameters['reduce_search_space_mode']}}\", \"--component_id={{$.pipeline_task_uuid}}\", \"--training_base_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/train\", \"--study_spec_override={{$.inputs.parameters['study_spec_override']}}\", \"--worker_pool_specs_override={{$.inputs.parameters['worker_pool_specs_override']}}\", \"--num_parallel_trial={{$.inputs.parameters['num_parallel_trials']}}\", \"--single_run_max_secs={{$.inputs.parameters['single_run_max_secs']}}\", \"--deadline_hours={{$.inputs.parameters['deadline_hours']}}\", \"--num_selected_trials={{$.inputs.parameters['num_selected_trials']}}\", \"--lro_job_info={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/lro\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--metadata_path={{$.inputs.artifacts['metadata'].uri}}\", \"--materialized_train_split={{$.inputs.artifacts['materialized_train_split'].uri}}\", \"--materialized_eval_split={{$.inputs.artifacts['materialized_eval_split'].uri}}\", \"--is_distill={{$.inputs.parameters['run_distillation']}}\", \"--tuning_result_output_path={{$.outputs.artifacts['tuning_result_output'].uri}}\", \"--kms_key_name={{$.inputs.parameters['encryption_spec_key_name']}}\", \"--use_json=true\", \"--log_level=ERROR\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-automl-tabular-transform": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"transform\", \"--transform_output_artifact_path={{$.outputs.artifacts['transform_output'].uri}}\", \"--transform_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", \"--materialized_splits_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\", \"--metadata_path={{$.inputs.artifacts['metadata'].uri}}\", \"--dataset_schema_path={{$.inputs.artifacts['dataset_schema'].uri}}\", \"--train_split={{$.inputs.artifacts['train_split'].uri}}\", \"--eval_split={{$.inputs.artifacts['eval_split'].uri}}\", \"--test_split={{$.inputs.artifacts['test_split'].uri}}\", \"--materialized_train_split={{$.outputs.artifacts['materialized_train_split'].uri}}\", \"--materialized_eval_split={{$.outputs.artifacts['materialized_eval_split'].uri}}\", \"--materialized_test_split={{$.outputs.artifacts['materialized_test_split'].uri}}\", \"--training_schema_path={{$.outputs.artifacts['training_schema_uri'].uri}}\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"--dataflow_project={{$.inputs.parameters['project']}}\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--dataflow_staging_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", \"--dataflow_tmp_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers={{$.inputs.parameters['dataflow_max_num_workers']}}\", \"--dataflow_machine_type={{$.inputs.parameters['dataflow_machine_type']}}\", \"--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:prod\", \"--dataflow_disk_size_gb={{$.inputs.parameters['dataflow_disk_size_gb']}}\", \"--dataflow_subnetwork_fully_qualified={{$.inputs.parameters['dataflow_subnetwork']}}\", \"--dataflow_use_public_ips={{$.inputs.parameters['dataflow_use_public_ips']}}\", \"--dataflow_kms_key={{$.inputs.parameters['encryption_spec_key_name']}}\", \"--dataflow_service_account={{$.inputs.parameters['dataflow_service_account']}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-automl-tabular-transform-2": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"transform\", \"--transform_output_artifact_path={{$.outputs.artifacts['transform_output'].uri}}\", \"--transform_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform\", \"--materialized_splits_output_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform_materialized\", \"--metadata_path={{$.inputs.artifacts['metadata'].uri}}\", \"--dataset_schema_path={{$.inputs.artifacts['dataset_schema'].uri}}\", \"--train_split={{$.inputs.artifacts['train_split'].uri}}\", \"--eval_split={{$.inputs.artifacts['eval_split'].uri}}\", \"--test_split={{$.inputs.artifacts['test_split'].uri}}\", \"--materialized_train_split={{$.outputs.artifacts['materialized_train_split'].uri}}\", \"--materialized_eval_split={{$.outputs.artifacts['materialized_eval_split'].uri}}\", \"--materialized_test_split={{$.outputs.artifacts['materialized_test_split'].uri}}\", \"--training_schema_path={{$.outputs.artifacts['training_schema_uri'].uri}}\", \"--job_name=automl-tabular-transform-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"--dataflow_project={{$.inputs.parameters['project']}}\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--dataflow_staging_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", \"--dataflow_tmp_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers={{$.inputs.parameters['dataflow_max_num_workers']}}\", \"--dataflow_machine_type={{$.inputs.parameters['dataflow_machine_type']}}\", \"--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:prod\", \"--dataflow_disk_size_gb={{$.inputs.parameters['dataflow_disk_size_gb']}}\", \"--dataflow_subnetwork_fully_qualified={{$.inputs.parameters['dataflow_subnetwork']}}\", \"--dataflow_use_public_ips={{$.inputs.parameters['dataflow_use_public_ips']}}\", \"--dataflow_kms_key={{$.inputs.parameters['encryption_spec_key_name']}}\", \"--dataflow_service_account={{$.inputs.parameters['dataflow_service_account']}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-bool-identity": { + "container": { + "args": [ + "--value", + "{{$.inputs.parameters['value']}}", + "----output-paths", + "{{$.outputs.parameters['Output'].output_file}}" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", + "def _bool_identity(value):\n \"\"\"Returns boolean value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\ndef _deserialize_bool(s) -> bool:\n from distutils.util import strtobool\n return strtobool(s) == 1\n\ndef _serialize_str(str_value: str) -> str:\n if not isinstance(str_value, str):\n raise TypeError('Value \"{}\" has type \"{}\" instead of str.'.format(\n str(str_value), str(type(str_value))))\n return str_value\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Bool identity', description='Returns boolean value.')\n_parser.add_argument(\"--value\", dest=\"value\", type=_deserialize_bool, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"----output-paths\", dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = _bool_identity(**_parsed_args)\n\n_outputs = [_outputs]\n\n_output_serializers = [\n _serialize_str,\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except OSError:\n pass\n with open(output_file, 'w') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n" + ], + "image": "python:3.7-slim" + } + }, + "exec-bool-identity-2": { + "container": { + "args": [ + "--value", + "{{$.inputs.parameters['value']}}", + "----output-paths", + "{{$.outputs.parameters['Output'].output_file}}" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", + "def _bool_identity(value):\n \"\"\"Returns boolean value.\n\n Args:\n value: Boolean value to return\n\n Returns:\n Boolean value.\n \"\"\"\n return 'true' if value else 'false'\n\ndef _deserialize_bool(s) -> bool:\n from distutils.util import strtobool\n return strtobool(s) == 1\n\ndef _serialize_str(str_value: str) -> str:\n if not isinstance(str_value, str):\n raise TypeError('Value \"{}\" has type \"{}\" instead of str.'.format(\n str(str_value), str(type(str_value))))\n return str_value\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Bool identity', description='Returns boolean value.')\n_parser.add_argument(\"--value\", dest=\"value\", type=_deserialize_bool, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"----output-paths\", dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = _bool_identity(**_parsed_args)\n\n_outputs = [_outputs]\n\n_output_serializers = [\n _serialize_str,\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except OSError:\n pass\n with open(output_file, 'w') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n" + ], + "image": "python:3.7-slim" + } + }, + "exec-merge-materialized-splits": { + "container": { + "args": [ + "--split-0", + "{{$.inputs.artifacts['split_0'].path}}", + "--split-1", + "{{$.inputs.artifacts['split_1'].path}}", + "--splits", + "{{$.outputs.artifacts['splits'].path}}" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", + "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef _merge_materialized_splits(\n split_0,\n split_1,\n splits,\n):\n \"\"\"Merge two materialized splits.\n\n Args:\n split_0: The first materialized split.\n split_1: The second materialized split.\n splits: The merged materialized split.\n \"\"\"\n with open(split_0, 'r') as f:\n split_0_content = f.read()\n with open(split_1, 'r') as f:\n split_1_content = f.read()\n with open(splits, 'w') as f:\n f.write(','.join([split_0_content, split_1_content]))\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Merge materialized splits', description='Merge two materialized splits.')\n_parser.add_argument(\"--split-0\", dest=\"split_0\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--split-1\", dest=\"split_1\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--splits\", dest=\"splits\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = _merge_materialized_splits(**_parsed_args)\n" + ], + "image": "python:3.7-slim" + } + }, + "exec-model-batch-explanation": { + "container": { + "args": [ + "--type", + "BatchPredictionJob", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['job_display_name']}}\", \"input_config\": {\"instances_format\": \"{{$.inputs.parameters['instances_format']}}\", \"gcs_source\": {\"uris\":{{$.inputs.parameters['gcs_source_uris']}}}, \"bigquery_source\": {\"input_uri\": \"{{$.inputs.parameters['bigquery_source_input_uri']}}\"}}, \"model_parameters\": {{$.inputs.parameters['model_parameters']}}, \"output_config\": {\"predictions_format\": \"{{$.inputs.parameters['predictions_format']}}\", \"gcs_destination\": {\"output_uri_prefix\": \"{{$.inputs.parameters['gcs_destination_output_uri_prefix']}}\"}, \"bigquery_destination\": {\"output_uri\": \"{{$.inputs.parameters['bigquery_destination_output_uri']}}\"}}, \"dedicated_resources\": {\"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\", \"accelerator_type\": \"{{$.inputs.parameters['accelerator_type']}}\", \"accelerator_count\": {{$.inputs.parameters['accelerator_count']}}}, \"starting_replica_count\": {{$.inputs.parameters['starting_replica_count']}}, \"max_replica_count\": {{$.inputs.parameters['max_replica_count']}}}, \"manual_batch_tuning_parameters\": {\"batch_size\": {{$.inputs.parameters['manual_batch_tuning_parameters_batch_size']}}}, \"generate_explanation\": {{$.inputs.parameters['generate_explanation']}}, \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"explanation_metadata_artifact\": \"{{$.inputs.artifacts['explanation_metadata_artifact'].uri}}\", \"labels\": {{$.inputs.parameters['labels']}}, \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "launcher" + ], + "image": "gcr.io/ml-pipeline/automl-tables-private:1.0.13" + } + }, + "exec-model-batch-explanation-2": { + "container": { + "args": [ + "--type", + "BatchPredictionJob", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['job_display_name']}}\", \"input_config\": {\"instances_format\": \"{{$.inputs.parameters['instances_format']}}\", \"gcs_source\": {\"uris\":{{$.inputs.parameters['gcs_source_uris']}}}, \"bigquery_source\": {\"input_uri\": \"{{$.inputs.parameters['bigquery_source_input_uri']}}\"}}, \"model_parameters\": {{$.inputs.parameters['model_parameters']}}, \"output_config\": {\"predictions_format\": \"{{$.inputs.parameters['predictions_format']}}\", \"gcs_destination\": {\"output_uri_prefix\": \"{{$.inputs.parameters['gcs_destination_output_uri_prefix']}}\"}, \"bigquery_destination\": {\"output_uri\": \"{{$.inputs.parameters['bigquery_destination_output_uri']}}\"}}, \"dedicated_resources\": {\"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\", \"accelerator_type\": \"{{$.inputs.parameters['accelerator_type']}}\", \"accelerator_count\": {{$.inputs.parameters['accelerator_count']}}}, \"starting_replica_count\": {{$.inputs.parameters['starting_replica_count']}}, \"max_replica_count\": {{$.inputs.parameters['max_replica_count']}}}, \"manual_batch_tuning_parameters\": {\"batch_size\": {{$.inputs.parameters['manual_batch_tuning_parameters_batch_size']}}}, \"generate_explanation\": {{$.inputs.parameters['generate_explanation']}}, \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"explanation_metadata_artifact\": \"{{$.inputs.artifacts['explanation_metadata_artifact'].uri}}\", \"labels\": {{$.inputs.parameters['labels']}}, \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "launcher" + ], + "image": "gcr.io/ml-pipeline/automl-tables-private:1.0.13" + } + }, + "exec-model-batch-predict": { + "container": { + "args": [ + "--type", + "BatchPredictionJob", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['job_display_name']}}\", \"input_config\": {\"instances_format\": \"{{$.inputs.parameters['instances_format']}}\", \"gcs_source\": {\"uris\":{{$.inputs.parameters['gcs_source_uris']}}}, \"bigquery_source\": {\"input_uri\": \"{{$.inputs.parameters['bigquery_source_input_uri']}}\"}}, \"model_parameters\": {{$.inputs.parameters['model_parameters']}}, \"output_config\": {\"predictions_format\": \"{{$.inputs.parameters['predictions_format']}}\", \"gcs_destination\": {\"output_uri_prefix\": \"{{$.inputs.parameters['gcs_destination_output_uri_prefix']}}\"}, \"bigquery_destination\": {\"output_uri\": \"{{$.inputs.parameters['bigquery_destination_output_uri']}}\"}}, \"dedicated_resources\": {\"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\", \"accelerator_type\": \"{{$.inputs.parameters['accelerator_type']}}\", \"accelerator_count\": {{$.inputs.parameters['accelerator_count']}}}, \"starting_replica_count\": {{$.inputs.parameters['starting_replica_count']}}, \"max_replica_count\": {{$.inputs.parameters['max_replica_count']}}}, \"manual_batch_tuning_parameters\": {\"batch_size\": {{$.inputs.parameters['manual_batch_tuning_parameters_batch_size']}}}, \"generate_explanation\": {{$.inputs.parameters['generate_explanation']}}, \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"labels\": {{$.inputs.parameters['labels']}}, \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-model-batch-predict-2": { + "container": { + "args": [ + "--type", + "BatchPredictionJob", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['job_display_name']}}\", \"model\": \"{{$.inputs.artifacts['model'].metadata['resourceName']}}\", \"input_config\": {\"instances_format\": \"{{$.inputs.parameters['instances_format']}}\", \"gcs_source\": {\"uris\":{{$.inputs.parameters['gcs_source_uris']}}}, \"bigquery_source\": {\"input_uri\": \"{{$.inputs.parameters['bigquery_source_input_uri']}}\"}}, \"model_parameters\": {{$.inputs.parameters['model_parameters']}}, \"output_config\": {\"predictions_format\": \"{{$.inputs.parameters['predictions_format']}}\", \"gcs_destination\": {\"output_uri_prefix\": \"{{$.inputs.parameters['gcs_destination_output_uri_prefix']}}\"}, \"bigquery_destination\": {\"output_uri\": \"{{$.inputs.parameters['bigquery_destination_output_uri']}}\"}}, \"dedicated_resources\": {\"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\", \"accelerator_type\": \"{{$.inputs.parameters['accelerator_type']}}\", \"accelerator_count\": {{$.inputs.parameters['accelerator_count']}}}, \"starting_replica_count\": {{$.inputs.parameters['starting_replica_count']}}, \"max_replica_count\": {{$.inputs.parameters['max_replica_count']}}}, \"manual_batch_tuning_parameters\": {\"batch_size\": {{$.inputs.parameters['manual_batch_tuning_parameters_batch_size']}}}, \"generate_explanation\": {{$.inputs.parameters['generate_explanation']}}, \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"labels\": {{$.inputs.parameters['labels']}}, \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-model-batch-predict-3": { + "container": { + "args": [ + "--type", + "BatchPredictionJob", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['job_display_name']}}\", \"model\": \"{{$.inputs.artifacts['model'].metadata['resourceName']}}\", \"input_config\": {\"instances_format\": \"{{$.inputs.parameters['instances_format']}}\", \"gcs_source\": {\"uris\":{{$.inputs.parameters['gcs_source_uris']}}}, \"bigquery_source\": {\"input_uri\": \"{{$.inputs.parameters['bigquery_source_input_uri']}}\"}}, \"model_parameters\": {{$.inputs.parameters['model_parameters']}}, \"output_config\": {\"predictions_format\": \"{{$.inputs.parameters['predictions_format']}}\", \"gcs_destination\": {\"output_uri_prefix\": \"{{$.inputs.parameters['gcs_destination_output_uri_prefix']}}\"}, \"bigquery_destination\": {\"output_uri\": \"{{$.inputs.parameters['bigquery_destination_output_uri']}}\"}}, \"dedicated_resources\": {\"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\", \"accelerator_type\": \"{{$.inputs.parameters['accelerator_type']}}\", \"accelerator_count\": {{$.inputs.parameters['accelerator_count']}}}, \"starting_replica_count\": {{$.inputs.parameters['starting_replica_count']}}, \"max_replica_count\": {{$.inputs.parameters['max_replica_count']}}}, \"manual_batch_tuning_parameters\": {\"batch_size\": {{$.inputs.parameters['manual_batch_tuning_parameters_batch_size']}}}, \"generate_explanation\": {{$.inputs.parameters['generate_explanation']}}, \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"labels\": {{$.inputs.parameters['labels']}}, \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-model-batch-predict-4": { + "container": { + "args": [ + "--type", + "BatchPredictionJob", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['job_display_name']}}\", \"input_config\": {\"instances_format\": \"{{$.inputs.parameters['instances_format']}}\", \"gcs_source\": {\"uris\":{{$.inputs.parameters['gcs_source_uris']}}}, \"bigquery_source\": {\"input_uri\": \"{{$.inputs.parameters['bigquery_source_input_uri']}}\"}}, \"model_parameters\": {{$.inputs.parameters['model_parameters']}}, \"output_config\": {\"predictions_format\": \"{{$.inputs.parameters['predictions_format']}}\", \"gcs_destination\": {\"output_uri_prefix\": \"{{$.inputs.parameters['gcs_destination_output_uri_prefix']}}\"}, \"bigquery_destination\": {\"output_uri\": \"{{$.inputs.parameters['bigquery_destination_output_uri']}}\"}}, \"dedicated_resources\": {\"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\", \"accelerator_type\": \"{{$.inputs.parameters['accelerator_type']}}\", \"accelerator_count\": {{$.inputs.parameters['accelerator_count']}}}, \"starting_replica_count\": {{$.inputs.parameters['starting_replica_count']}}, \"max_replica_count\": {{$.inputs.parameters['max_replica_count']}}}, \"manual_batch_tuning_parameters\": {\"batch_size\": {{$.inputs.parameters['manual_batch_tuning_parameters_batch_size']}}}, \"generate_explanation\": {{$.inputs.parameters['generate_explanation']}}, \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"labels\": {{$.inputs.parameters['labels']}}, \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.batch_prediction_job.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-model-evaluation": { + "container": { + "args": [ + "--setup_file", + "/setup.py", + "--json_mode", + "true", + "--project_id", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--problem_type", + "{{$.inputs.parameters['problem_type']}}", + "--batch_prediction_format", + "{{$.inputs.parameters['predictions_format']}}", + "--batch_prediction_gcs_source", + "{{$.inputs.artifacts['batch_prediction_job'].metadata['gcsOutputDirectory']}}", + "--ground_truth_format", + "{{$.inputs.parameters['ground_truth_format']}}", + "--ground_truth_gcs_source", + "{{$.inputs.parameters['ground_truth_gcs_source']}}", + "--key_prefix_in_prediction_dataset", + "instance", + "--key_columns", + "{{$.inputs.parameters['key_columns']}}", + "--root_dir", + "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--classification_type", + "{{$.inputs.parameters['classification_type']}}", + "--class_names", + "{{$.inputs.parameters['class_names']}}", + "--ground_truth_column", + "instance.{{$.inputs.parameters['ground_truth_column']}}", + "--prediction_score_column", + "{{$.inputs.parameters['prediction_score_column']}}", + "--prediction_label_column", + "{{$.inputs.parameters['prediction_label_column']}}", + "--prediction_id_column", + "{{$.inputs.parameters['prediction_id_column']}}", + "--example_weight_column", + "{{$.inputs.parameters['example_weight_column']}}", + "--positive_classes", + "{{$.inputs.parameters['positive_classes']}}", + "--generate_feature_attribution", + "{{$.inputs.parameters['generate_feature_attribution']}}", + "--dataflow_job_prefix", + "evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--dataflow_service_account", + "{{$.inputs.parameters['dataflow_service_account']}}", + "--dataflow_disk_size", + "{{$.inputs.parameters['dataflow_disk_size']}}", + "--dataflow_machine_type", + "{{$.inputs.parameters['dataflow_machine_type']}}", + "--dataflow_workers_num", + "{{$.inputs.parameters['dataflow_workers_num']}}", + "--dataflow_max_workers_num", + "{{$.inputs.parameters['dataflow_max_workers_num']}}", + "--dataflow_subnetwork", + "{{$.inputs.parameters['dataflow_subnetwork']}}", + "--dataflow_use_public_ips", + "{{$.inputs.parameters['dataflow_use_public_ips']}}", + "--kms_key_name", + "{{$.inputs.parameters['encryption_spec_key_name']}}", + "--output_metrics_gcs_path", + "{{$.outputs.artifacts['evaluation_metrics'].uri}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python", + "/main.py" + ], + "image": "gcr.io/ml-pipeline/model-evaluation:v0.4" + } + }, + "exec-model-evaluation-2": { + "container": { + "args": [ + "--setup_file", + "/setup.py", + "--json_mode", + "true", + "--project_id", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--problem_type", + "{{$.inputs.parameters['problem_type']}}", + "--batch_prediction_format", + "{{$.inputs.parameters['predictions_format']}}", + "--batch_prediction_gcs_source", + "{{$.inputs.artifacts['batch_prediction_job'].metadata['gcsOutputDirectory']}}", + "--ground_truth_format", + "{{$.inputs.parameters['ground_truth_format']}}", + "--ground_truth_gcs_source", + "{{$.inputs.parameters['ground_truth_gcs_source']}}", + "--key_prefix_in_prediction_dataset", + "instance", + "--key_columns", + "{{$.inputs.parameters['key_columns']}}", + "--root_dir", + "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--classification_type", + "{{$.inputs.parameters['classification_type']}}", + "--class_names", + "{{$.inputs.parameters['class_names']}}", + "--ground_truth_column", + "instance.{{$.inputs.parameters['ground_truth_column']}}", + "--prediction_score_column", + "{{$.inputs.parameters['prediction_score_column']}}", + "--prediction_label_column", + "{{$.inputs.parameters['prediction_label_column']}}", + "--prediction_id_column", + "{{$.inputs.parameters['prediction_id_column']}}", + "--example_weight_column", + "{{$.inputs.parameters['example_weight_column']}}", + "--positive_classes", + "{{$.inputs.parameters['positive_classes']}}", + "--generate_feature_attribution", + "{{$.inputs.parameters['generate_feature_attribution']}}", + "--dataflow_job_prefix", + "evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--dataflow_service_account", + "{{$.inputs.parameters['dataflow_service_account']}}", + "--dataflow_disk_size", + "{{$.inputs.parameters['dataflow_disk_size']}}", + "--dataflow_machine_type", + "{{$.inputs.parameters['dataflow_machine_type']}}", + "--dataflow_workers_num", + "{{$.inputs.parameters['dataflow_workers_num']}}", + "--dataflow_max_workers_num", + "{{$.inputs.parameters['dataflow_max_workers_num']}}", + "--dataflow_subnetwork", + "{{$.inputs.parameters['dataflow_subnetwork']}}", + "--dataflow_use_public_ips", + "{{$.inputs.parameters['dataflow_use_public_ips']}}", + "--kms_key_name", + "{{$.inputs.parameters['encryption_spec_key_name']}}", + "--output_metrics_gcs_path", + "{{$.outputs.artifacts['evaluation_metrics'].uri}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python", + "/main.py" + ], + "image": "gcr.io/ml-pipeline/model-evaluation:v0.4" + } + }, + "exec-model-evaluation-3": { + "container": { + "args": [ + "--setup_file", + "/setup.py", + "--json_mode", + "true", + "--project_id", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--problem_type", + "{{$.inputs.parameters['problem_type']}}", + "--batch_prediction_format", + "{{$.inputs.parameters['predictions_format']}}", + "--batch_prediction_gcs_source", + "{{$.inputs.artifacts['batch_prediction_job'].metadata['gcsOutputDirectory']}}", + "--ground_truth_format", + "{{$.inputs.parameters['ground_truth_format']}}", + "--ground_truth_gcs_source", + "{{$.inputs.parameters['ground_truth_gcs_source']}}", + "--key_prefix_in_prediction_dataset", + "instance", + "--key_columns", + "{{$.inputs.parameters['key_columns']}}", + "--root_dir", + "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--classification_type", + "{{$.inputs.parameters['classification_type']}}", + "--class_names", + "{{$.inputs.parameters['class_names']}}", + "--ground_truth_column", + "instance.{{$.inputs.parameters['ground_truth_column']}}", + "--prediction_score_column", + "{{$.inputs.parameters['prediction_score_column']}}", + "--prediction_label_column", + "{{$.inputs.parameters['prediction_label_column']}}", + "--prediction_id_column", + "{{$.inputs.parameters['prediction_id_column']}}", + "--example_weight_column", + "{{$.inputs.parameters['example_weight_column']}}", + "--positive_classes", + "{{$.inputs.parameters['positive_classes']}}", + "--generate_feature_attribution", + "{{$.inputs.parameters['generate_feature_attribution']}}", + "--dataflow_job_prefix", + "evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--dataflow_service_account", + "{{$.inputs.parameters['dataflow_service_account']}}", + "--dataflow_disk_size", + "{{$.inputs.parameters['dataflow_disk_size']}}", + "--dataflow_machine_type", + "{{$.inputs.parameters['dataflow_machine_type']}}", + "--dataflow_workers_num", + "{{$.inputs.parameters['dataflow_workers_num']}}", + "--dataflow_max_workers_num", + "{{$.inputs.parameters['dataflow_max_workers_num']}}", + "--dataflow_subnetwork", + "{{$.inputs.parameters['dataflow_subnetwork']}}", + "--dataflow_use_public_ips", + "{{$.inputs.parameters['dataflow_use_public_ips']}}", + "--kms_key_name", + "{{$.inputs.parameters['encryption_spec_key_name']}}", + "--output_metrics_gcs_path", + "{{$.outputs.artifacts['evaluation_metrics'].uri}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python", + "/main.py" + ], + "image": "gcr.io/ml-pipeline/model-evaluation:v0.4" + } + }, + "exec-model-evaluation-4": { + "container": { + "args": [ + "--setup_file", + "/setup.py", + "--json_mode", + "true", + "--project_id", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--problem_type", + "{{$.inputs.parameters['problem_type']}}", + "--batch_prediction_format", + "{{$.inputs.parameters['predictions_format']}}", + "--batch_prediction_gcs_source", + "{{$.inputs.artifacts['batch_prediction_job'].metadata['gcsOutputDirectory']}}", + "--ground_truth_format", + "{{$.inputs.parameters['ground_truth_format']}}", + "--ground_truth_gcs_source", + "{{$.inputs.parameters['ground_truth_gcs_source']}}", + "--key_prefix_in_prediction_dataset", + "instance", + "--key_columns", + "{{$.inputs.parameters['key_columns']}}", + "--root_dir", + "{{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--classification_type", + "{{$.inputs.parameters['classification_type']}}", + "--class_names", + "{{$.inputs.parameters['class_names']}}", + "--ground_truth_column", + "instance.{{$.inputs.parameters['ground_truth_column']}}", + "--prediction_score_column", + "{{$.inputs.parameters['prediction_score_column']}}", + "--prediction_label_column", + "{{$.inputs.parameters['prediction_label_column']}}", + "--prediction_id_column", + "{{$.inputs.parameters['prediction_id_column']}}", + "--example_weight_column", + "{{$.inputs.parameters['example_weight_column']}}", + "--positive_classes", + "{{$.inputs.parameters['positive_classes']}}", + "--generate_feature_attribution", + "{{$.inputs.parameters['generate_feature_attribution']}}", + "--dataflow_job_prefix", + "evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}", + "--dataflow_service_account", + "{{$.inputs.parameters['dataflow_service_account']}}", + "--dataflow_disk_size", + "{{$.inputs.parameters['dataflow_disk_size']}}", + "--dataflow_machine_type", + "{{$.inputs.parameters['dataflow_machine_type']}}", + "--dataflow_workers_num", + "{{$.inputs.parameters['dataflow_workers_num']}}", + "--dataflow_max_workers_num", + "{{$.inputs.parameters['dataflow_max_workers_num']}}", + "--dataflow_subnetwork", + "{{$.inputs.parameters['dataflow_subnetwork']}}", + "--dataflow_use_public_ips", + "{{$.inputs.parameters['dataflow_use_public_ips']}}", + "--kms_key_name", + "{{$.inputs.parameters['encryption_spec_key_name']}}", + "--output_metrics_gcs_path", + "{{$.outputs.artifacts['evaluation_metrics'].uri}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python", + "/main.py" + ], + "image": "gcr.io/ml-pipeline/model-evaluation:v0.4" + } + }, + "exec-model-evaluation-import": { + "container": { + "args": [ + "--metrics", + "{{$.inputs.artifacts['metrics'].uri}}", + "--metrics_explanation", + "{{$.inputs.artifacts['metrics'].metadata['explanation_gcs_path']}}", + "--explanation", + "{{$.inputs.artifacts['explanation'].metadata['explanation_gcs_path']}}", + "--problem_type", + "{{$.inputs.parameters['problem_type']}}", + "--display_name", + "{{$.inputs.parameters['display_name']}}", + "--dataset_path", + "{{$.inputs.parameters['dataset_path']}}", + "--dataset_paths", + "{{$.inputs.parameters['dataset_paths']}}", + "--dataset_type", + "{{$.inputs.parameters['dataset_type']}}", + "--pipeline_job_id", + "{{$.pipeline_job_uuid}}", + "--pipeline_job_resource_name", + "{{$.pipeline_job_resource_name}}", + "--model_name", + "{{$.inputs.artifacts['model'].metadata['resourceName']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.experimental.evaluation.import_model_evaluation" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-model-evaluation-import-2": { + "container": { + "args": [ + "--metrics", + "{{$.inputs.artifacts['metrics'].uri}}", + "--metrics_explanation", + "{{$.inputs.artifacts['metrics'].metadata['explanation_gcs_path']}}", + "--explanation", + "{{$.inputs.artifacts['explanation'].metadata['explanation_gcs_path']}}", + "--problem_type", + "{{$.inputs.parameters['problem_type']}}", + "--display_name", + "{{$.inputs.parameters['display_name']}}", + "--dataset_path", + "{{$.inputs.parameters['dataset_path']}}", + "--dataset_paths", + "{{$.inputs.parameters['dataset_paths']}}", + "--dataset_type", + "{{$.inputs.parameters['dataset_type']}}", + "--pipeline_job_id", + "{{$.pipeline_job_uuid}}", + "--pipeline_job_resource_name", + "{{$.pipeline_job_resource_name}}", + "--model_name", + "{{$.inputs.artifacts['model'].metadata['resourceName']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.experimental.evaluation.import_model_evaluation" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-model-upload": { + "container": { + "args": [ + "--type", + "UploadModel", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['display_name']}}\", \"description\": \"{{$.inputs.parameters['description']}}\", \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"explanation_metadata_artifact\": \"{{$.inputs.artifacts['explanation_metadata_artifact'].uri}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"labels\": {{$.inputs.parameters['labels']}}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "launcher" + ], + "image": "gcr.io/ml-pipeline/automl-tables-private:1.0.13" + } + }, + "exec-model-upload-2": { + "container": { + "args": [ + "--type", + "UploadModel", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['display_name']}}\", \"description\": \"{{$.inputs.parameters['description']}}\", \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"explanation_metadata_artifact\": \"{{$.inputs.artifacts['explanation_metadata_artifact'].uri}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"labels\": {{$.inputs.parameters['labels']}}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "launcher" + ], + "image": "gcr.io/ml-pipeline/automl-tables-private:1.0.13" + } + }, + "exec-model-upload-3": { + "container": { + "args": [ + "--type", + "UploadModel", + "--payload", + "{\"display_name\": \"{{$.inputs.parameters['display_name']}}\", \"description\": \"{{$.inputs.parameters['description']}}\", \"explanation_spec\": {\"parameters\": {{$.inputs.parameters['explanation_parameters']}}, \"metadata\": {{$.inputs.parameters['explanation_metadata']}}}, \"explanation_metadata_artifact\": \"{{$.inputs.artifacts['explanation_metadata_artifact'].uri}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"labels\": {{$.inputs.parameters['labels']}}}", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--executor_input", + "{{$}}" + ], + "command": [ + "python3", + "-u", + "-m", + "launcher" + ], + "image": "gcr.io/ml-pipeline/automl-tables-private:1.0.13" + } + }, + "exec-read-input-uri": { + "container": { + "args": [ + "--split-uri", + "{{$.inputs.artifacts['split_uri'].path}}", + "----output-paths", + "{{$.outputs.parameters['Output'].output_file}}" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", + "def _read_input_uri(split_uri):\n \"\"\"Construct Dataset based on the batch prediction job.\n\n Args:\n split_uri: Tbe path to the file that contains Dataset data.\n\n Returns:\n The list of string that represents the batch prediction input files.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n return data_source['tf_record_data_source']['file_patterns']\n\ndef _serialize_json(obj) -> str:\n if isinstance(obj, str):\n return obj\n import json\n\n def default_serializer(obj):\n if hasattr(obj, 'to_struct'):\n return obj.to_struct()\n else:\n raise TypeError(\n \"Object of type '%s' is not JSON serializable and does not have .to_struct() method.\"\n % obj.__class__.__name__)\n\n return json.dumps(obj, default=default_serializer, sort_keys=True)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Read input uri', description='Construct Dataset based on the batch prediction job.')\n_parser.add_argument(\"--split-uri\", dest=\"split_uri\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"----output-paths\", dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = _read_input_uri(**_parsed_args)\n\n_outputs = [_outputs]\n\n_output_serializers = [\n _serialize_json,\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except OSError:\n pass\n with open(output_file, 'w') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n" + ], + "image": "python:3.7-slim" + } + }, + "exec-read-input-uri-2": { + "container": { + "args": [ + "--split-uri", + "{{$.inputs.artifacts['split_uri'].path}}", + "----output-paths", + "{{$.outputs.parameters['Output'].output_file}}" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", + "def _read_input_uri(split_uri):\n \"\"\"Construct Dataset based on the batch prediction job.\n\n Args:\n split_uri: Tbe path to the file that contains Dataset data.\n\n Returns:\n The list of string that represents the batch prediction input files.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n with open(split_uri, 'r') as f:\n data_source = json.loads(f.read())\n return data_source['tf_record_data_source']['file_patterns']\n\ndef _serialize_json(obj) -> str:\n if isinstance(obj, str):\n return obj\n import json\n\n def default_serializer(obj):\n if hasattr(obj, 'to_struct'):\n return obj.to_struct()\n else:\n raise TypeError(\n \"Object of type '%s' is not JSON serializable and does not have .to_struct() method.\"\n % obj.__class__.__name__)\n\n return json.dumps(obj, default=default_serializer, sort_keys=True)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Read input uri', description='Construct Dataset based on the batch prediction job.')\n_parser.add_argument(\"--split-uri\", dest=\"split_uri\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"----output-paths\", dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = _read_input_uri(**_parsed_args)\n\n_outputs = [_outputs]\n\n_output_serializers = [\n _serialize_json,\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except OSError:\n pass\n with open(output_file, 'w') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n" + ], + "image": "python:3.7-slim" + } + }, + "exec-set-model-can-skip-validation": { + "container": { + "args": [ + "--executor_input", + "{{$}}", + "--function_to_execute", + "_set_model_can_skip_validation" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", + "\nimport kfp\nfrom kfp.v2 import dsl\nfrom kfp.v2.dsl import *\nfrom typing import *\n\ndef _set_model_can_skip_validation(model: Input[Artifact]):\n \"\"\"Construct Dataset based on the batch prediction job.\n\n Args:\n model: The model artifact.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n import os\n import tensorflow as tf\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\n # create an empty CAN_SKIP_VALIDATION file\n with tf.io.gfile.GFile(os.path.join(model.uri, 'CAN_SKIP_VALIDATION'),\n 'w') as f:\n f.write('')\n\n" + ], + "image": "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:prod" + } + }, + "exec-tabular-stats-and-example-gen": { + "container": { + "args": [ + "--type", + "CustomJob", + "--project", + "{{$.inputs.parameters['project']}}", + "--location", + "{{$.inputs.parameters['location']}}", + "--gcp_resources", + "{{$.outputs.parameters['gcp_resources'].output_file}}", + "--payload", + "{\"display_name\": \"tabular-stats-and-example-gen-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"encryption_spec\": {\"kms_key_name\":\"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": 1, \"machine_spec\": {\"machine_type\": \"n1-standard-8\"}, \"container_spec\": {\"image_uri\":\"us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:prod\", \"args\": [\"stats_generator\",\"--train_spec={\\\"prediction_type\\\": \\\"{{$.inputs.parameters['prediction_type']}}\\\", \\\"target_column\\\": \\\"{{$.inputs.parameters['target_column_name']}}\\\", \\\"optimization_objective\\\": \\\"{{$.inputs.parameters['optimization_objective']}}\\\", \\\"weight_column_name\\\": \\\"{{$.inputs.parameters['weight_column_name']}}\\\", \\\"transformations\\\": {{$.inputs.parameters['transformations']}}}\", \"--transformations_override_path={{$.inputs.parameters['transformations_path']}}\", \"--split_spec={{$.inputs.parameters['split_spec']}}\", \"--data_source={{$.inputs.parameters['data_source']}}\", \"--data_source_csv_filenames={{$.inputs.parameters['data_source_csv_filenames']}}\", \"--data_source_bigquery_table_path={{$.inputs.parameters['data_source_bigquery_table_path']}}\", \"--predefined_split_key={{$.inputs.parameters['predefined_split_key']}}\", \"--timestamp_split_key={{$.inputs.parameters['timestamp_split_key']}}\", \"--stratified_split_key={{$.inputs.parameters['stratified_split_key']}}\", \"--training_fraction={{$.inputs.parameters['training_fraction']}}\", \"--validation_fraction={{$.inputs.parameters['validation_fraction']}}\", \"--test_fraction={{$.inputs.parameters['test_fraction']}}\", \"--target_column={{$.inputs.parameters['target_column_name']}}\", \"--request_type={{$.inputs.parameters['request_type']}}\", \"--optimization_objective_recall_value={{$.inputs.parameters['optimization_objective_recall_value']}}\", \"--optimization_objective_precision_value={{$.inputs.parameters['optimization_objective_precision_value']}}\", \"--example_gen_gcs_output_prefix={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/example_gen_output\", \"--dataset_stats_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/stats/\", \"--stats_result_path={{$.outputs.artifacts['dataset_stats'].uri}}\", \"--dataset_schema_path={{$.outputs.artifacts['dataset_schema'].uri}}\", \"--job_name=tabular-stats-and-example-gen-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}\", \"--dataflow_project={{$.inputs.parameters['project']}}\", \"--error_file_path={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.pb\", \"--dataflow_staging_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging\", \"--dataflow_tmp_dir={{$.inputs.parameters['root_dir']}}/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp\", \"--dataflow_max_num_workers={{$.inputs.parameters['dataflow_max_num_workers']}}\", \"--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:prod\", \"--dataflow_machine_type={{$.inputs.parameters['dataflow_machine_type']}}\", \"--dataflow_disk_size_gb={{$.inputs.parameters['dataflow_disk_size_gb']}}\", \"--dataflow_kms_key={{$.inputs.parameters['encryption_spec_key_name']}}\", \"--dataflow_subnetwork_fully_qualified={{$.inputs.parameters['dataflow_subnetwork']}}\", \"--dataflow_use_public_ips={{$.inputs.parameters['dataflow_use_public_ips']}}\", \"--dataflow_service_account={{$.inputs.parameters['dataflow_service_account']}}\", \"--is_distill={{$.inputs.parameters['run_distillation']}}\", \"--additional_experiments={{$.inputs.parameters['additional_experiments']}}\", \"--metadata_path={{$.outputs.artifacts['metadata'].uri}}\", \"--train_split={{$.outputs.artifacts['train_split'].uri}}\", \"--eval_split={{$.outputs.artifacts['eval_split'].uri}}\", \"--test_split={{$.outputs.artifacts['test_split'].uri}}\", \"--test_split_for_batch_prediction_component={{$.outputs.parameters['test_split_json'].output_file}}\", \"--downsampled_test_split_for_batch_prediction_component={{$.outputs.parameters['downsampled_test_split_json'].output_file}}\", \"--instance_baseline_path={{$.outputs.artifacts['instance_baseline'].uri}}\", \"--parse_json=true\", \"--generate_additional_downsample_test_split=true\", \"--executor_input={{$.json_escape[1]}}\"]}}]}}" + ], + "command": [ + "python3", + "-u", + "-m", + "google_cloud_pipeline_components.container.v1.gcp_launcher.launcher" + ], + "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.21" + } + }, + "exec-write-bp-result-path": { + "container": { + "args": [ + "--executor_input", + "{{$}}", + "--function_to_execute", + "_write_bp_result_path" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", + "\nimport kfp\nfrom kfp.v2 import dsl\nfrom kfp.v2.dsl import *\nfrom typing import *\n\ndef _write_bp_result_path(\n bp_job: Input[Artifact],\n result: OutputPath('Dataset'),\n):\n \"\"\"Construct Dataset based on the batch prediction job.\n\n Args:\n bp_job: The batch prediction job artifact.\n result: Tbe path to the file that contains Dataset data.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n directory = bp_job.metadata['gcsOutputDirectory']\n data_source = {\n 'tf_record_data_source': {\n 'file_patterns': [f'{directory}/prediction.results-*',],\n 'coder': 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\n" + ], + "image": "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:prod" + } + }, + "exec-write-bp-result-path-2": { + "container": { + "args": [ + "--executor_input", + "{{$}}", + "--function_to_execute", + "_write_bp_result_path" + ], + "command": [ + "sh", + "-ec", + "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.v2.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", + "\nimport kfp\nfrom kfp.v2 import dsl\nfrom kfp.v2.dsl import *\nfrom typing import *\n\ndef _write_bp_result_path(\n bp_job: Input[Artifact],\n result: OutputPath('Dataset'),\n):\n \"\"\"Construct Dataset based on the batch prediction job.\n\n Args:\n bp_job: The batch prediction job artifact.\n result: Tbe path to the file that contains Dataset data.\n \"\"\"\n # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n import json\n # pylint: enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n directory = bp_job.metadata['gcsOutputDirectory']\n data_source = {\n 'tf_record_data_source': {\n 'file_patterns': [f'{directory}/prediction.results-*',],\n 'coder': 'PROTO_VALUE',\n },\n }\n with open(result, 'w') as f:\n f.write(json.dumps(data_source))\n\n" + ], + "image": "us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:prod" + } + } + } + }, + "pipelineInfo": { + "name": "automl-tabular-deprecated" + }, + "root": { + "dag": { + "outputs": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-2-evaluation_metrics", + "producerSubtask": "exit-handler-1" + } + ] + }, + "model-evaluation-3-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-3-evaluation_metrics", + "producerSubtask": "exit-handler-1" + } + ] + }, + "model-evaluation-4-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-4-evaluation_metrics", + "producerSubtask": "exit-handler-1" + } + ] + }, + "model-evaluation-evaluation_metrics": { + "artifactSelectors": [ + { + "outputArtifactKey": "model-evaluation-evaluation_metrics", + "producerSubtask": "exit-handler-1" + } + ] + } + } + }, + "tasks": { + "automl-tabular-finalizer": { + "componentRef": { + "name": "comp-automl-tabular-finalizer" + }, + "dependentTasks": [ + "exit-handler-1" + ], + "inputs": { + "parameters": { + "encryption_spec_key_name": { + "runtimeValue": { + "constantValue": { + "stringValue": "" + } + } + }, + "location": { + "componentInputParameter": "location" + }, + "project": { + "componentInputParameter": "project" + }, + "root_dir": { + "componentInputParameter": "root_dir" + } + } + }, + "taskInfo": { + "name": "automl-tabular-finalizer" + }, + "triggerPolicy": { + "strategy": "ALL_UPSTREAM_TASKS_COMPLETED" + } + }, + "exit-handler-1": { + "componentRef": { + "name": "comp-exit-handler-1" + }, + "inputs": { + "parameters": { + "pipelineparam--additional_experiments": { + "componentInputParameter": "additional_experiments" + }, + "pipelineparam--cv_trainer_worker_pool_specs_override": { + "componentInputParameter": "cv_trainer_worker_pool_specs_override" + }, + "pipelineparam--data_source": { + "componentInputParameter": "data_source" + }, + "pipelineparam--dataflow_service_account": { + "componentInputParameter": "dataflow_service_account" + }, + "pipelineparam--dataflow_subnetwork": { + "componentInputParameter": "dataflow_subnetwork" + }, + "pipelineparam--dataflow_use_public_ips": { + "componentInputParameter": "dataflow_use_public_ips" + }, + "pipelineparam--disable_early_stopping": { + "componentInputParameter": "disable_early_stopping" + }, + "pipelineparam--distill_batch_predict_machine_type": { + "componentInputParameter": "distill_batch_predict_machine_type" + }, + "pipelineparam--distill_batch_predict_max_replica_count": { + "componentInputParameter": "distill_batch_predict_max_replica_count" + }, + "pipelineparam--distill_batch_predict_starting_replica_count": { + "componentInputParameter": "distill_batch_predict_starting_replica_count" + }, + "pipelineparam--distill_stage_1_deadline_hours": { + "componentInputParameter": "distill_stage_1_deadline_hours" + }, + "pipelineparam--encryption_spec_key_name": { + "componentInputParameter": "encryption_spec_key_name" + }, + "pipelineparam--evaluation_batch_predict_machine_type": { + "componentInputParameter": "evaluation_batch_predict_machine_type" + }, + "pipelineparam--evaluation_batch_predict_max_replica_count": { + "componentInputParameter": "evaluation_batch_predict_max_replica_count" + }, + "pipelineparam--evaluation_batch_predict_starting_replica_count": { + "componentInputParameter": "evaluation_batch_predict_starting_replica_count" + }, + "pipelineparam--evaluation_dataflow_disk_size_gb": { + "componentInputParameter": "evaluation_dataflow_disk_size_gb" + }, + "pipelineparam--evaluation_dataflow_machine_type": { + "componentInputParameter": "evaluation_dataflow_machine_type" + }, + "pipelineparam--evaluation_dataflow_max_num_workers": { + "componentInputParameter": "evaluation_dataflow_max_num_workers" + }, + "pipelineparam--export_additional_model_without_custom_ops": { + "componentInputParameter": "export_additional_model_without_custom_ops" + }, + "pipelineparam--location": { + "componentInputParameter": "location" + }, + "pipelineparam--optimization_objective": { + "componentInputParameter": "optimization_objective" + }, + "pipelineparam--optimization_objective_precision_value": { + "componentInputParameter": "optimization_objective_precision_value" + }, + "pipelineparam--optimization_objective_recall_value": { + "componentInputParameter": "optimization_objective_recall_value" + }, + "pipelineparam--prediction_type": { + "componentInputParameter": "prediction_type" + }, + "pipelineparam--project": { + "componentInputParameter": "project" + }, + "pipelineparam--reduce_search_space_mode": { + "componentInputParameter": "reduce_search_space_mode" + }, + "pipelineparam--root_dir": { + "componentInputParameter": "root_dir" + }, + "pipelineparam--run_distillation": { + "componentInputParameter": "run_distillation" + }, + "pipelineparam--run_evaluation": { + "componentInputParameter": "run_evaluation" + }, + "pipelineparam--split_spec": { + "componentInputParameter": "split_spec" + }, + "pipelineparam--stage_1_deadline_hours": { + "componentInputParameter": "stage_1_deadline_hours" + }, + "pipelineparam--stage_1_num_parallel_trials": { + "componentInputParameter": "stage_1_num_parallel_trials" + }, + "pipelineparam--stage_1_num_selected_trials": { + "componentInputParameter": "stage_1_num_selected_trials" + }, + "pipelineparam--stage_1_single_run_max_secs": { + "componentInputParameter": "stage_1_single_run_max_secs" + }, + "pipelineparam--stage_1_tuner_worker_pool_specs_override": { + "componentInputParameter": "stage_1_tuner_worker_pool_specs_override" + }, + "pipelineparam--stage_2_deadline_hours": { + "componentInputParameter": "stage_2_deadline_hours" + }, + "pipelineparam--stage_2_num_parallel_trials": { + "componentInputParameter": "stage_2_num_parallel_trials" + }, + "pipelineparam--stage_2_num_selected_trials": { + "componentInputParameter": "stage_2_num_selected_trials" + }, + "pipelineparam--stage_2_single_run_max_secs": { + "componentInputParameter": "stage_2_single_run_max_secs" + }, + "pipelineparam--stats_and_example_gen_dataflow_disk_size_gb": { + "componentInputParameter": "stats_and_example_gen_dataflow_disk_size_gb" + }, + "pipelineparam--stats_and_example_gen_dataflow_machine_type": { + "componentInputParameter": "stats_and_example_gen_dataflow_machine_type" + }, + "pipelineparam--stats_and_example_gen_dataflow_max_num_workers": { + "componentInputParameter": "stats_and_example_gen_dataflow_max_num_workers" + }, + "pipelineparam--study_spec_override": { + "componentInputParameter": "study_spec_override" + }, + "pipelineparam--target_column_name": { + "componentInputParameter": "target_column_name" + }, + "pipelineparam--transform_dataflow_disk_size_gb": { + "componentInputParameter": "transform_dataflow_disk_size_gb" + }, + "pipelineparam--transform_dataflow_machine_type": { + "componentInputParameter": "transform_dataflow_machine_type" + }, + "pipelineparam--transform_dataflow_max_num_workers": { + "componentInputParameter": "transform_dataflow_max_num_workers" + }, + "pipelineparam--transformations": { + "componentInputParameter": "transformations" + }, + "pipelineparam--weight_column_name": { + "componentInputParameter": "weight_column_name" + } + } + }, + "taskInfo": { + "name": "exit-handler-1" + } + } + } + }, + "inputDefinitions": { + "parameters": { + "additional_experiments": { + "type": "STRING" + }, + "cv_trainer_worker_pool_specs_override": { + "type": "STRING" + }, + "data_source": { + "type": "STRING" + }, + "dataflow_service_account": { + "type": "STRING" + }, + "dataflow_subnetwork": { + "type": "STRING" + }, + "dataflow_use_public_ips": { + "type": "STRING" + }, + "disable_early_stopping": { + "type": "STRING" + }, + "distill_batch_predict_machine_type": { + "type": "STRING" + }, + "distill_batch_predict_max_replica_count": { + "type": "INT" + }, + "distill_batch_predict_starting_replica_count": { + "type": "INT" + }, + "distill_stage_1_deadline_hours": { + "type": "DOUBLE" + }, + "encryption_spec_key_name": { + "type": "STRING" + }, + "evaluation_batch_predict_machine_type": { + "type": "STRING" + }, + "evaluation_batch_predict_max_replica_count": { + "type": "INT" + }, + "evaluation_batch_predict_starting_replica_count": { + "type": "INT" + }, + "evaluation_dataflow_disk_size_gb": { + "type": "INT" + }, + "evaluation_dataflow_machine_type": { + "type": "STRING" + }, + "evaluation_dataflow_max_num_workers": { + "type": "INT" + }, + "export_additional_model_without_custom_ops": { + "type": "STRING" + }, + "location": { + "type": "STRING" + }, + "optimization_objective": { + "type": "STRING" + }, + "optimization_objective_precision_value": { + "type": "DOUBLE" + }, + "optimization_objective_recall_value": { + "type": "DOUBLE" + }, + "prediction_type": { + "type": "STRING" + }, + "project": { + "type": "STRING" + }, + "reduce_search_space_mode": { + "type": "STRING" + }, + "root_dir": { + "type": "STRING" + }, + "run_distillation": { + "type": "STRING" + }, + "run_evaluation": { + "type": "STRING" + }, + "split_spec": { + "type": "STRING" + }, + "stage_1_deadline_hours": { + "type": "DOUBLE" + }, + "stage_1_num_parallel_trials": { + "type": "INT" + }, + "stage_1_num_selected_trials": { + "type": "INT" + }, + "stage_1_single_run_max_secs": { + "type": "INT" + }, + "stage_1_tuner_worker_pool_specs_override": { + "type": "STRING" + }, + "stage_2_deadline_hours": { + "type": "DOUBLE" + }, + "stage_2_num_parallel_trials": { + "type": "INT" + }, + "stage_2_num_selected_trials": { + "type": "INT" + }, + "stage_2_single_run_max_secs": { + "type": "INT" + }, + "stats_and_example_gen_dataflow_disk_size_gb": { + "type": "INT" + }, + "stats_and_example_gen_dataflow_machine_type": { + "type": "STRING" + }, + "stats_and_example_gen_dataflow_max_num_workers": { + "type": "INT" + }, + "study_spec_override": { + "type": "STRING" + }, + "target_column_name": { + "type": "STRING" + }, + "transform_dataflow_disk_size_gb": { + "type": "INT" + }, + "transform_dataflow_machine_type": { + "type": "STRING" + }, + "transform_dataflow_max_num_workers": { + "type": "INT" + }, + "transformations": { + "type": "STRING" + }, + "weight_column_name": { + "type": "STRING" + } + } + }, + "outputDefinitions": { + "artifacts": { + "model-evaluation-2-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-3-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-4-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + }, + "model-evaluation-evaluation_metrics": { + "artifactType": { + "schemaTitle": "system.Metrics", + "schemaVersion": "0.0.1" + } + } + } + } + }, + "schemaVersion": "2.0.0", + "sdkVersion": "kfp-1.8.11" + }, + "runtimeConfig": { + "parameters": { + "additional_experiments": { + "stringValue": "" + }, + "cv_trainer_worker_pool_specs_override": { + "stringValue": "" + }, + "dataflow_service_account": { + "stringValue": "" + }, + "dataflow_subnetwork": { + "stringValue": "" + }, + "dataflow_use_public_ips": { + "stringValue": "True" + }, + "disable_early_stopping": { + "stringValue": "False" + }, + "distill_batch_predict_machine_type": { + "stringValue": "n1-standard-16" + }, + "distill_batch_predict_max_replica_count": { + "intValue": "25" + }, + "distill_batch_predict_starting_replica_count": { + "intValue": "25" + }, + "distill_stage_1_deadline_hours": { + "doubleValue": 1.0 + }, + "encryption_spec_key_name": { + "stringValue": "" + }, + "evaluation_batch_predict_machine_type": { + "stringValue": "n1-standard-16" + }, + "evaluation_batch_predict_max_replica_count": { + "intValue": "25" + }, + "evaluation_batch_predict_starting_replica_count": { + "intValue": "25" + }, + "evaluation_dataflow_disk_size_gb": { + "intValue": "50" + }, + "evaluation_dataflow_machine_type": { + "stringValue": "n1-standard-4" + }, + "evaluation_dataflow_max_num_workers": { + "intValue": "25" + }, + "export_additional_model_without_custom_ops": { + "stringValue": "False" + }, + "optimization_objective_precision_value": { + "doubleValue": -1.0 + }, + "optimization_objective_recall_value": { + "doubleValue": -1.0 + }, + "reduce_search_space_mode": { + "stringValue": "regular" + }, + "run_distillation": { + "stringValue": "False" + }, + "run_evaluation": { + "stringValue": "False" + }, + "stage_1_tuner_worker_pool_specs_override": { + "stringValue": "" + }, + "stats_and_example_gen_dataflow_disk_size_gb": { + "intValue": "40" + }, + "stats_and_example_gen_dataflow_machine_type": { + "stringValue": "n1-standard-16" + }, + "stats_and_example_gen_dataflow_max_num_workers": { + "intValue": "25" + }, + "study_spec_override": { + "stringValue": "" + }, + "transform_dataflow_disk_size_gb": { + "intValue": "40" + }, + "transform_dataflow_machine_type": { + "stringValue": "n1-standard-16" + }, + "transform_dataflow_max_num_workers": { + "intValue": "25" + }, + "weight_column_name": { + "stringValue": "" + } + } + } +} \ No newline at end of file diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/ensemble.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/ensemble.py new file mode 100644 index 0000000000..453050dcb7 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/ensemble.py @@ -0,0 +1,160 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Tabular Ensemble component spec.""" + +from typing import Optional + +from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Dataset +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.container_component +def automl_tabular_ensemble( + project: str, + location: str, + root_dir: str, + transform_output: Input[Artifact], + metadata: Input[Artifact], + dataset_schema: Input[Artifact], + tuning_result_input: Input[Artifact], + instance_baseline: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + model_architecture: Output[Artifact], + model: Output[Artifact], + unmanaged_container_model: Output[UnmanagedContainerModel], + model_without_custom_ops: Output[Artifact], + explanation_metadata: dsl.OutputPath(dict), + explanation_metadata_artifact: Output[Artifact], + explanation_parameters: dsl.OutputPath(dict), + warmup_data: Optional[Input[Dataset]] = None, + encryption_spec_key_name: Optional[str] = '', + export_additional_model_without_custom_ops: Optional[bool] = False, +): + # fmt: off + """Ensembles AutoML Tabular models. + + Args: + project: Project to run Cross-validation trainer. + location: Location for running the Cross-validation trainer. + root_dir: The Cloud Storage location to store the output. + transform_output: The transform output artifact. + metadata: The tabular example gen metadata. + dataset_schema: The schema of the dataset. + tuning_result_input: AutoML Tabular tuning result. + instance_baseline: The instance baseline used to calculate explanations. + warmup_data: The warm up data. Ensemble component will save the warm up data together with the model artifact, used to warm up the model when prediction server starts. + encryption_spec_key_name: Customer-managed encryption key. + export_additional_model_without_custom_ops: True if export an additional model without custom TF operators to the `model_without_custom_ops` output. + + Returns: + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + model_architecture: The architecture of the output model. + model: The output model. + model_without_custom_ops: The output model without custom TF operators, this output will be empty unless `export_additional_model_without_custom_ops` is set. + model_uri: The URI of the output model. + instance_schema_uri: The URI of the instance schema. + prediction_schema_uri: The URI of the prediction schema. + explanation_metadata: The explanation metadata used by Vertex online and batch explanations. + explanation_metadata: The explanation parameters used by Vertex online and batch explanations. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "automl-tabular-ensemble-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-highmem-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "args": ["ensemble", "--transform_output_path=', + transform_output.uri, + '", "--model_output_path=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/model",' + ' "--custom_model_output_path=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/custom_model",' + ' "--error_file_path=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb",' + ' "--export_custom_model=' + ), + export_additional_model_without_custom_ops, + '", "--metadata_path=', + metadata.uri, + '", "--dataset_schema_path=', + dataset_schema.uri, + '", "--tuning_result_input_path=', + tuning_result_input.uri, + '", "--instance_baseline_path=', + instance_baseline.uri, + '", "--warmup_data=', + warmup_data.uri, + '", "--prediction_docker_uri=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125', + '", "--model_path=', + model.uri, + '", "--custom_model_path=', + model_without_custom_ops.uri, + '", "--explanation_metadata_path=', + explanation_metadata, + ',', + explanation_metadata_artifact.uri, + '", "--explanation_parameters_path=', + explanation_parameters, + '", "--model_architecture_path=', + model_architecture.uri, + ( + '", "--use_json=true",' + ' "--executor_input={{$.json_escape[1]}}"]}}]}}' + ), + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/finalizer.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/finalizer.py new file mode 100644 index 0000000000..d272b5be4e --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/finalizer.py @@ -0,0 +1,87 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Pipeline Finalizer component spec.""" + +from typing import Optional + +from kfp import dsl + + +@dsl.container_component +def automl_tabular_finalizer( + project: str, + location: str, + root_dir: str, + gcp_resources: dsl.OutputPath(str), + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Finalizes AutoML Tabular pipelines. + + Args: + project: Project to run Cross-validation trainer. + location: Location for running the Cross-validation trainer. + root_dir: The Cloud Storage location to store the output. + encryption_spec_key_name: Customer-managed encryption key. + + Returns: + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "automl-tabular-finalizer-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "args": ["cancel_l2l_tuner", "--error_file_path=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb",' + ' "--cleanup_lro_job_infos=' + ), + root_dir, + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/lro"' + ']}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py new file mode 100644 index 0000000000..ef9c5b38f7 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/infra_validator.py @@ -0,0 +1,38 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Infra Validator component spec.""" + +from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel +from kfp import dsl +from kfp.dsl import Input + + +@dsl.container_component +def automl_tabular_infra_validator( + unmanaged_container_model: Input[UnmanagedContainerModel], # pylint: disable=unused-argument +): + # fmt: off + """Validates the trained AutoML Tabular model is a valid model. + + Args: + unmanaged_container_model: google.UnmanagedContainerModel for model to be validated. + """ + # fmt: on + + return dsl.ContainerSpec( + image='us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:20231002_0125', + command=[], + args=['--executor_input', '{{$}}'], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py new file mode 100644 index 0000000000..90ab7709f0 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/split_materialized_data.py @@ -0,0 +1,119 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Split Materialized Data component spec.""" + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Dataset +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.container_component +def split_materialized_data( + materialized_data: Input[Dataset], + materialized_train_split: Output[Artifact], + materialized_eval_split: Output[Artifact], + materialized_test_split: Output[Artifact], +): + # fmt: off + """Splits materialized dataset into train, eval, and test data splits. + + The materialized dataset generated by the Feature Transform Engine consists of + all the splits + that were combined into the input transform dataset (i.e., train, eval, and + test splits). + This components splits the output materialized dataset into corresponding + materialized data splits + so that the splits can be used by down-stream training or evaluation + components. + + Args: + materialized_data: Materialized dataset output by the Feature + Transform Engine. + + Returns: + materialized_train_split: Path patern to materialized train split. + materialized_eval_split: Path patern to materialized eval split. + materialized_test_split: Path patern to materialized test split. + """ + # fmt: on + + return dsl.ContainerSpec( + image='us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125', + command=[ + 'sh', + '-ec', + ( + 'program_path=$(mktemp -d)\nprintf "%s" "$0" >' + ' "$program_path/ephemeral_component.py"\npython3 -m' + ' kfp.components.executor_main ' + ' --component_module_path ' + ' "$program_path/ephemeral_component.py" ' + ' "$@"\n' + ), + ( + '\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom' + ' typing import *\n\ndef _split_materialized_data(\n ' + ' materialized_data: Input[Dataset],\n ' + " materialized_train_split: OutputPath('MaterializedSplit'),\n " + " materialized_eval_split: OutputPath('MaterializedSplit'),\n " + " materialized_test_split: OutputPath('MaterializedSplit')):\n " + ' """Splits materialized_data into materialized_data test,' + ' train, and eval splits.\n\n Necessary adapter between FTE' + ' pipeline and trainer.\n\n Args:\n materialized_data:' + ' materialized_data dataset output by FTE.\n ' + ' materialized_train_split: Path patern to' + ' materialized_train_split.\n materialized_eval_split: Path' + ' patern to materialized_eval_split.\n ' + ' materialized_test_split: Path patern to' + ' materialized_test_split.\n """\n # pylint:' + ' disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n' + ' import json\n import tensorflow as tf\n # pylint:' + ' enable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported\n\n' + " with tf.io.gfile.GFile(materialized_data.path, 'r') as f:\n " + ' artifact_path = f.read()\n\n # needed to import tf because' + ' this is a path in gs://\n with' + " tf.io.gfile.GFile(artifact_path, 'r') as f:\n " + ' materialized_data_json = json.load(f)\n\n if' + " 'tf_record_data_source' in materialized_data_json:\n " + ' file_patterns =' + " materialized_data_json['tf_record_data_source'][\n " + " 'file_patterns']\n elif 'avro_data_source' in" + ' materialized_data_json:\n file_patterns =' + " materialized_data_json['avro_data_source'][\n " + " 'file_patterns']\n elif 'parquet_data_source' in" + ' materialized_data_json:\n file_patterns =' + " materialized_data_json['parquet_data_source'][\n " + " 'file_patterns']\n else:\n raise ValueError(f'Unsupported" + " training data source: {materialized_data_json}')\n\n # we map" + ' indices to file patterns based on the ordering of insertion' + ' order\n # in our transform_data (see above in' + ' _generate_analyze_and_transform_data)\n with' + " tf.io.gfile.GFile(materialized_train_split, 'w') as f:\n " + ' f.write(file_patterns[0])\n\n with' + " tf.io.gfile.GFile(materialized_eval_split, 'w') as f:\n " + ' f.write(file_patterns[1])\n\n with' + " tf.io.gfile.GFile(materialized_test_split, 'w') as f:\n " + ' f.write(file_patterns[2])\n\n' + ), + ], + args=[ + '--executor_input', + '{{$}}', + '--function_to_execute', + '_split_materialized_data', + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py new file mode 100644 index 0000000000..21dd5d48f2 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stage_1_tuner.py @@ -0,0 +1,177 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Tabular Stage 1 Tuner component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.container_component +def automl_tabular_stage_1_tuner( + project: str, + location: str, + root_dir: str, + num_selected_trials: int, + deadline_hours: float, + num_parallel_trials: int, + single_run_max_secs: int, + metadata: Input[Artifact], + transform_output: Input[Artifact], + materialized_train_split: Input[Artifact], + materialized_eval_split: Input[Artifact], + gcp_resources: dsl.OutputPath(str), + tuning_result_output: Output[Artifact], + execution_metrics: dsl.OutputPath(dict), + study_spec_parameters_override: Optional[list] = [], + worker_pool_specs_override_json: Optional[list] = [], + reduce_search_space_mode: Optional[str] = 'regular', + num_selected_features: Optional[int] = 0, + disable_early_stopping: Optional[bool] = False, + feature_ranking: Optional[Input[Artifact]] = None, + tune_feature_selection_rate: Optional[bool] = False, + encryption_spec_key_name: Optional[str] = '', + run_distillation: Optional[bool] = False, +): + # fmt: off + """Searches AutoML Tabular architectures and selects the top trials. + + Args: + project: Project to run Cross-validation trainer. + location: Location for running the Cross-validation trainer. + root_dir: The Cloud Storage location to store the output. + study_spec_parameters_override: JSON study spec. E.g., [{"parameter_id": "model_type","categorical_value_spec": {"values": ["nn"]}}] + worker_pool_specs_override_json: JSON worker pool specs. E.g., [{"machine_spec": {"machine_type": "n1-standard-16"}},{},{},{"machine_spec": {"machine_type": "n1-standard-16"}}] + reduce_search_space_mode: The reduce search space mode. Possible + values: "regular" (default), "minimal", "full". + num_selected_trials: Number of selected trials. The number of weak learners in the final model is 5 * num_selected_trials. + num_selected_features: Number of selected features. The number of features to learn in the NN models. + deadline_hours: Number of hours the cross-validation trainer should run. + disable_early_stopping: True if disable early stopping. Default value is false. + num_parallel_trials: Number of parallel training trials. + single_run_max_secs: Max number of seconds each training trial runs. + metadata: The tabular example gen metadata. + transform_output: The transform output artifact. + materialized_train_split: The materialized train split. + materialized_eval_split: The materialized eval split. + encryption_spec_key_name: Customer-managed encryption key. + run_distillation: True if in distillation mode. The default value is false. + + Returns: + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + tuning_result_output: The trained model and architectures. + execution_metrics: Core metrics in dictionary of component execution. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "automl-tabular-stage-1-tuner-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "args": ["l2l_stage_1_tuner", "--transform_output_path=', + transform_output.uri, + '", "--training_docker_uri=', + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "--feature_selection_result_path=', + feature_ranking.uri, + '", "--disable_early_stopping=', + disable_early_stopping, + '", "--tune_feature_selection_rate=', + tune_feature_selection_rate, + '", "--reduce_search_space_mode=', + reduce_search_space_mode, + ( + f'", "--component_id={dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "--training_base_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/train",' + ' "--num_parallel_trial=' + ), + num_parallel_trials, + '", "--single_run_max_secs=', + single_run_max_secs, + '", "--deadline_hours=', + deadline_hours, + '", "--num_selected_trials=', + num_selected_trials, + '", "--num_selected_features=', + num_selected_features, + '", "--lro_job_info=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/lro",' + ' "--error_file_path=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb",' + ' "--metadata_path=' + ), + metadata.uri, + '", "--materialized_train_split=', + materialized_train_split.uri, + '", "--materialized_eval_split=', + materialized_eval_split.uri, + '", "--is_distill=', + run_distillation, + '", "--tuning_result_output_path=', + tuning_result_output.uri, + '", "--kms_key_name=', + encryption_spec_key_name, + '", "--gcp_resources_path=', + gcp_resources, + '", "--execution_metrics_path=', + execution_metrics, + ( + '", "--use_json=true", "--log_level=ERROR",' + ' "--executor_input={{$.json_escape[1]}}"]}}]}}' + ), + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py new file mode 100644 index 0000000000..7f1aee2194 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/stats_and_example_gen.py @@ -0,0 +1,269 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Stats and Example Generation component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Dataset +from kfp.dsl import Output + + +@dsl.container_component +def tabular_stats_and_example_gen( + project: str, + location: str, + root_dir: str, + target_column_name: str, + prediction_type: str, + transformations: str, + dataset_schema: Output[Artifact], + dataset_stats: Output[Artifact], + train_split: Output[Dataset], + eval_split: Output[Dataset], + test_split: Output[Dataset], + test_split_json: dsl.OutputPath(list), + downsampled_test_split_json: dsl.OutputPath(list), + instance_baseline: Output[Artifact], + metadata: Output[Artifact], + gcp_resources: dsl.OutputPath(str), + weight_column_name: Optional[str] = '', + optimization_objective: Optional[str] = '', + optimization_objective_recall_value: Optional[float] = -1, + optimization_objective_precision_value: Optional[float] = -1, + transformations_path: Optional[str] = '', + request_type: Optional[str] = 'COLUMN_STATS_ONLY', + dataflow_machine_type: Optional[str] = 'n1-standard-16', + dataflow_max_num_workers: Optional[int] = 25, + dataflow_disk_size_gb: Optional[int] = 40, + dataflow_subnetwork: Optional[str] = '', + dataflow_use_public_ips: Optional[bool] = True, + dataflow_service_account: Optional[str] = '', + encryption_spec_key_name: Optional[str] = '', + run_distillation: Optional[bool] = False, + additional_experiments: Optional[str] = '', + additional_experiments_json: Optional[dict] = {}, + data_source_csv_filenames: Optional[str] = '', + data_source_bigquery_table_path: Optional[str] = '', + predefined_split_key: Optional[str] = '', + timestamp_split_key: Optional[str] = '', + stratified_split_key: Optional[str] = '', + training_fraction: Optional[float] = -1, + validation_fraction: Optional[float] = -1, + test_fraction: Optional[float] = -1, + quantiles: Optional[list] = [], + enable_probabilistic_inference: Optional[bool] = False, +): + # fmt: off + """Generates stats and training instances for tabular data. + + Args: + project: Project to run dataset statistics and example generation. + location: Location for running dataset statistics and example generation. + root_dir: The Cloud Storage location to store the output. + target_column_name: The target column name. + weight_column_name: The weight column name. + prediction_type: The prediction type. Supported values: "classification", "regression". + optimization_objective: Objective function the model is optimizing towards. The training process creates a model that maximizes/minimizes the value of the objective function over the validation set. The supported optimization objectives depend on the prediction type. If the field is not set, a default objective function is used. + classification: "maximize-au-roc" (default) - Maximize the area under the receiver operating characteristic (ROC) curve. "minimize-log-loss" - Minimize log loss. "maximize-au-prc" - Maximize the area under the precision-recall curve. "maximize-precision-at-recall" - Maximize precision for a specified recall value. "maximize-recall-at-precision" - Maximize recall for a specified precision value. classification (multi-class): "minimize-log-loss" (default) - Minimize log loss. + regression: "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). "minimize-mae" - Minimize mean-absolute error (MAE). "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). + optimization_objective_recall_value: Required when optimization_objective is "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + transformations: Quote escaped JSON string for transformations. Each transformation will apply transform function to given input column. And the result will be used for training. When creating transformation for BigQuery Struct column, the column should be flattened using "." as the delimiter. + transformations_path: Path to a GCS file containing JSON string for transformations. + dataflow_machine_type: The machine type used for dataflow jobs. If not set, default to n1-standard-16. + dataflow_max_num_workers: The number of workers to run the dataflow job. If not set, default to 25. + dataflow_disk_size_gb: The disk size, in gigabytes, to use on each Dataflow worker instance. If not set, default to 40. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More + details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + dataflow_service_account: Custom service account to run dataflow jobs. + encryption_spec_key_name: Customer-managed encryption key. + run_distillation: True if in distillation mode. The default value is false. + + Returns: + dataset_schema: The schema of the dataset. + dataset_stats: The stats of the dataset. + train_split: The train split. + eval_split: The eval split. + test_split: The test split. + test_split_json: The test split JSON object. + downsampled_test_split_json: The downsampled test split JSON object. + instance_baseline: The instance baseline used to calculate explanations. + metadata: The tabular example gen metadata. + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "tabular-stats-and-example-gen-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + '", "args": ["stats_generator",', + '"--train_spec={\\"prediction_type\\": \\"', + prediction_type, + '\\", \\"target_column\\": \\"', + target_column_name, + '\\", \\"optimization_objective\\": \\"', + optimization_objective, + '\\", \\"weight_column_name\\": \\"', + weight_column_name, + '\\", \\"transformations\\": ', + transformations, + ', \\"quantiles\\": ', + quantiles, + ', \\"enable_probabilistic_inference\\": ', + enable_probabilistic_inference, + '}", "--transformations_override_path=', + transformations_path, + '", "--data_source_csv_filenames=', + data_source_csv_filenames, + '", "--data_source_bigquery_table_path=', + data_source_bigquery_table_path, + '", "--predefined_split_key=', + predefined_split_key, + '", "--timestamp_split_key=', + timestamp_split_key, + '", "--stratified_split_key=', + stratified_split_key, + '", "--training_fraction=', + training_fraction, + '", "--validation_fraction=', + validation_fraction, + '", "--test_fraction=', + test_fraction, + '", "--target_column=', + target_column_name, + '", "--request_type=', + request_type, + '", "--optimization_objective_recall_value=', + optimization_objective_recall_value, + '", "--optimization_objective_precision_value=', + optimization_objective_precision_value, + '", "--example_gen_gcs_output_prefix=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/example_gen_output",' + ' "--dataset_stats_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/stats/",' + ' "--stats_result_path=' + ), + dataset_stats.uri, + '", "--dataset_schema_path=', + dataset_schema.uri, + ( + f'", "--job_name=tabular-stats-and-example-gen-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + '", "--dataflow_project=', + project, + '", "--error_file_path=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb",' + ' "--dataflow_staging_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_staging",' + ' "--dataflow_tmp_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_tmp",' + ' "--dataflow_max_num_workers=' + ), + dataflow_max_num_workers, + '", "--dataflow_worker_container_image=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125', + '", "--dataflow_machine_type=', + dataflow_machine_type, + '", "--dataflow_disk_size_gb=', + dataflow_disk_size_gb, + '", "--dataflow_kms_key=', + encryption_spec_key_name, + '", "--dataflow_subnetwork_fully_qualified=', + dataflow_subnetwork, + '", "--dataflow_use_public_ips=', + dataflow_use_public_ips, + '", "--dataflow_service_account=', + dataflow_service_account, + '", "--is_distill=', + run_distillation, + '", "--additional_experiments=', + additional_experiments, + '", "--metadata_path=', + metadata.uri, + '", "--train_split=', + train_split.uri, + '", "--eval_split=', + eval_split.uri, + '", "--test_split=', + test_split.uri, + '", "--test_split_for_batch_prediction_component=', + test_split_json, + ( + '", "--downsampled_test_split_for_batch_prediction_component=' + ), + downsampled_test_split_json, + '", "--instance_baseline_path=', + instance_baseline.uri, + '", "--lro_job_info=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/lro",' + ' "--gcp_resources_path=' + ), + gcp_resources, + ( + '", "--parse_json=true",' + ' "--generate_additional_downsample_test_split=true",' + ' "--executor_input={{$.json_escape[1]}}"]}}]}}' + ), + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py new file mode 100644 index 0000000000..13f5fda4ee --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/training_configurator_and_validator.py @@ -0,0 +1,249 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Training Configurator and Validator component spec.""" + +from typing import Optional + +from kfp import dsl + + +@dsl.container_component +def training_configurator_and_validator( + dataset_stats: dsl.Input[dsl.Artifact], + split_example_counts: str, + training_schema: dsl.Input[dsl.Artifact], + instance_schema: dsl.Input[dsl.Artifact], + metadata: dsl.Output[dsl.Artifact], + instance_baseline: dsl.Output[dsl.Artifact], + target_column: Optional[str] = '', + weight_column: Optional[str] = '', + prediction_type: Optional[str] = '', + optimization_objective: Optional[str] = '', + optimization_objective_recall_value: Optional[float] = -1, + optimization_objective_precision_value: Optional[float] = -1, + run_evaluation: Optional[bool] = False, + run_distill: Optional[bool] = False, + enable_probabilistic_inference: Optional[bool] = False, + time_series_identifier_column: Optional[str] = None, + time_series_identifier_columns: Optional[list] = [], + time_column: Optional[str] = '', + time_series_attribute_columns: Optional[list] = [], + available_at_forecast_columns: Optional[list] = [], + unavailable_at_forecast_columns: Optional[list] = [], + quantiles: Optional[list] = [], + context_window: Optional[int] = -1, + forecast_horizon: Optional[int] = -1, + forecasting_model_type: Optional[str] = '', + forecasting_transformations: Optional[dict] = {}, + stage_1_deadline_hours: Optional[float] = None, + stage_2_deadline_hours: Optional[float] = None, + group_columns: Optional[list] = None, + group_total_weight: float = 0.0, + temporal_total_weight: float = 0.0, + group_temporal_total_weight: float = 0.0, +): + # fmt: off + """Configures training and validates data and user-input configurations. + + Args: + dataset_stats: Dataset stats generated by feature transform engine. + split_example_counts: JSON string of data split example counts for train, validate, and test splits. + training_schema_path: Schema of input data to the tf_model at training time. + instance_schema: Schema of input data to the tf_model at serving time. + target_column: Target column of input data. + weight_column: Weight column of input data. + prediction_type: Model prediction type. One of "classification", "regression", "time_series". + optimization_objective: Objective function the model is optimizing towards. The training process creates a model that maximizes/minimizes the value of the objective function over the validation set. The supported optimization objectives depend on the prediction type. If the field is not set, a default objective function is used. + classification: "maximize-au-roc" (default) - Maximize the area under the receiver operating characteristic (ROC) curve. "minimize-log-loss" - Minimize log loss. "maximize-au-prc" - Maximize the area under the precision-recall curve. "maximize-precision-at-recall" - Maximize precision for a specified recall value. "maximize-recall-at-precision" - Maximize recall for a specified precision value. classification (multi-class): "minimize-log-loss" (default) - Minimize log loss. + regression: "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). "minimize-mae" - Minimize mean-absolute error (MAE). "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). + optimization_objective_recall_value: Required when optimization_objective is "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + run_evaluation: Whether we are running evaluation in the training pipeline. + run_distill: Whether the distillation should be applied to the training. + enable_probabilistic_inference: If probabilistic inference is enabled, the model will fit a distribution that captures the uncertainty of a prediction. At inference time, the predictive distribution is used to make a point prediction that minimizes the optimization objective. For example, the mean of a predictive distribution is the point prediction that minimizes RMSE loss. If quantiles are specified, then the quantiles of the distribution are also returned. + time_series_identifier_column: [Deprecated] The time series identifier column. Used by forecasting only. Raises exception if used - use the "time_series_identifier_column" field instead. + time_series_identifier_columns: The list of time series identifier columns. Used by forecasting only. + time_column: The column that indicates the time. Used by forecasting only. + time_series_attribute_columns: The column names of the time series attributes. + available_at_forecast_columns: The names of the columns that are available at forecast time. + unavailable_at_forecast_columns: The names of the columns that are not available at forecast time. + quantiles: All quantiles that the model need to predict. + context_window: The length of the context window. + forecast_horizon: The length of the forecast horizon. + forecasting_model_type: The model types, e.g. l2l, seq2seq, tft. + forecasting_transformations: Dict mapping auto and/or type-resolutions to feature columns. The supported types are auto, categorical, numeric, text, and timestamp. + stage_1_deadline_hours: Stage 1 training budget in hours. + stage_2_deadline_hours: Stage 2 training budget in hours. + group_columns: A list of time series attribute column names that define the time series hierarchy. + group_total_weight: The weight of the loss for predictions aggregated over time series in the same group. + temporal_total_weight: The weight of the loss for predictions aggregated over the horizon for a single time series. + group_temporal_total_weight: The weight of the loss for predictions aggregated over both the horizon and time series in the same hierarchy group. + + Returns: + metadata: The tabular example gen metadata. + """ + # fmt: on + + return dsl.ContainerSpec( + image='us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20231002_0125', + command=[], + args=[ + 'training_configurator_and_validator', + dsl.ConcatPlaceholder( + items=['--instance_schema_path=', instance_schema.uri] + ), + dsl.ConcatPlaceholder( + items=['--training_schema_path=', training_schema.uri] + ), + dsl.ConcatPlaceholder( + items=['--dataset_stats_path=', dataset_stats.uri] + ), + dsl.ConcatPlaceholder( + items=['--split_example_counts=', split_example_counts] + ), + dsl.ConcatPlaceholder(items=['--target_column=', target_column]), + dsl.ConcatPlaceholder(items=['--weight_column=', weight_column]), + dsl.ConcatPlaceholder(items=['--prediction_type=', prediction_type]), + dsl.ConcatPlaceholder( + items=['--optimization_objective=', optimization_objective] + ), + dsl.ConcatPlaceholder( + items=[ + '--optimization_objective_recall_value=', + optimization_objective_recall_value, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--optimization_objective_precision_value=', + optimization_objective_precision_value, + ] + ), + dsl.ConcatPlaceholder(items=['--metadata_path=', metadata.uri]), + dsl.ConcatPlaceholder( + items=['--instance_baseline_path=', instance_baseline.uri] + ), + dsl.ConcatPlaceholder(items=['--run_evaluation=', run_evaluation]), + dsl.ConcatPlaceholder(items=['--run_distill=', run_distill]), + dsl.ConcatPlaceholder( + items=[ + '--enable_probabilistic_inference=', + enable_probabilistic_inference, + ] + ), + dsl.IfPresentPlaceholder( + # Singular time series ID backwards support. + input_name='time_series_identifier_column', + then=dsl.ConcatPlaceholder( + items=[ + '--time_series_identifier_column=', + time_series_identifier_column, + ] + ), + ), + dsl.ConcatPlaceholder( + items=[ + '--time_series_identifier_columns=', + time_series_identifier_columns, + ] + ), + dsl.ConcatPlaceholder(items=['--time_column=', time_column]), + dsl.ConcatPlaceholder( + items=[ + '--time_series_attribute_columns=', + time_series_attribute_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--available_at_forecast_columns=', + available_at_forecast_columns, + ] + ), + dsl.ConcatPlaceholder( + items=[ + '--unavailable_at_forecast_columns=', + unavailable_at_forecast_columns, + ] + ), + dsl.IfPresentPlaceholder( + input_name='quantiles', + then=dsl.ConcatPlaceholder( + items=[ + '--quantiles=', + quantiles, + ] + ), + ), + dsl.ConcatPlaceholder(items=['--context_window=', context_window]), + dsl.ConcatPlaceholder( + items=['--forecast_horizon=', forecast_horizon] + ), + dsl.ConcatPlaceholder( + items=['--forecasting_model_type=', forecasting_model_type] + ), + dsl.ConcatPlaceholder( + items=[ + '--forecasting_transformations=', + forecasting_transformations, + ] + ), + dsl.IfPresentPlaceholder( + input_name='stage_1_deadline_hours', + then=dsl.ConcatPlaceholder( + items=[ + '--stage_1_deadline_hours=', + stage_1_deadline_hours, + ] + ), + ), + dsl.IfPresentPlaceholder( + input_name='stage_2_deadline_hours', + then=dsl.ConcatPlaceholder( + items=[ + '--stage_2_deadline_hours=', + stage_2_deadline_hours, + ] + ), + ), + dsl.IfPresentPlaceholder( + input_name='group_columns', + then=dsl.ConcatPlaceholder( + items=['--group_columns=', group_columns] + ), + ), + dsl.IfPresentPlaceholder( + input_name='group_total_weight', + then=dsl.ConcatPlaceholder( + items=['--group_total_weight=', group_total_weight] + ), + ), + dsl.IfPresentPlaceholder( + input_name='temporal_total_weight', + then=dsl.ConcatPlaceholder( + items=['--temporal_total_weight=', temporal_total_weight] + ), + ), + dsl.IfPresentPlaceholder( + input_name='group_temporal_total_weight', + then=dsl.ConcatPlaceholder( + items=[ + '--group_temporal_total_weight=', + group_temporal_total_weight, + ] + ), + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/transform.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/transform.py new file mode 100644 index 0000000000..3565816924 --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/transform.py @@ -0,0 +1,193 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AutoML Transform component spec.""" + +from typing import Optional + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Dataset +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.container_component +def automl_tabular_transform( + project: str, + location: str, + root_dir: str, + metadata: Input[Artifact], + dataset_schema: Input[Artifact], + train_split: Input[Dataset], + eval_split: Input[Dataset], + test_split: Input[Dataset], + materialized_train_split: Output[Artifact], + materialized_eval_split: Output[Artifact], + materialized_test_split: Output[Artifact], + training_schema_uri: Output[Artifact], + transform_output: Output[Artifact], + gcp_resources: dsl.OutputPath(str), + dataflow_machine_type: Optional[str] = 'n1-standard-16', + dataflow_max_num_workers: Optional[int] = 25, + dataflow_disk_size_gb: Optional[int] = 40, + dataflow_subnetwork: Optional[str] = '', + dataflow_use_public_ips: Optional[bool] = True, + dataflow_service_account: Optional[str] = '', + encryption_spec_key_name: Optional[str] = '', +): + # fmt: off + """Transforms raw features to engineered features. + + Args: + project: Project to run Cross-validation trainer. + location: Location for running the Cross-validation trainer. + root_dir: The Cloud Storage location to store the output. + metadata: The tabular example gen metadata. + dataset_schema: The schema of the dataset. + train_split: The train split. + eval_split: The eval split. + test_split: The test split. + dataflow_machine_type: The machine type used for dataflow jobs. If not set, default to n1-standard-16. + dataflow_max_num_workers: The number of workers to run the dataflow job. If not set, default to 25. + dataflow_disk_size_gb: The disk size, in gigabytes, to use on each Dataflow worker instance. If not set, default to 40. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More + details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + dataflow_service_account: Custom service account to run dataflow jobs. + encryption_spec_key_name: Customer-managed encryption key. + + Returns: + materialized_train_split: The materialized train split. + materialized_eval_split: The materialized eval split. + materialized_eval_split: The materialized test split. + training_schema_uri: The training schema. + transform_output: The transform output artifact. + gcp_resources: GCP resources created by this component. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + """ + # fmt: on + + return dsl.ContainerSpec( + image='gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.44', + command=[ + 'python3', + '-u', + '-m', + 'google_cloud_pipeline_components.container.v1.custom_job.launcher', + ], + args=[ + '--type', + 'CustomJob', + '--project', + project, + '--location', + location, + '--gcp_resources', + gcp_resources, + '--payload', + dsl.ConcatPlaceholder( + items=[ + ( + '{"display_name":' + f' "automl-tabular-transform-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}",' + ' "encryption_spec": {"kms_key_name":"' + ), + encryption_spec_key_name, + ( + '"}, "job_spec": {"worker_pool_specs": [{"replica_count":' + ' 1, "machine_spec": {"machine_type": "n1-standard-8"},' + ' "container_spec": {"image_uri":"' + ), + 'us-docker.pkg.dev/vertex-ai-restricted/automl-tabular/training:20231002_0125', + ( + '", "args": ["transform", "--is_mp=true",' + ' "--transform_output_artifact_path=' + ), + transform_output.uri, + '", "--transform_output_path=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/transform",' + ' "--materialized_splits_output_path=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/transform_materialized",' + ' "--metadata_path=' + ), + metadata.uri, + '", "--dataset_schema_path=', + dataset_schema.uri, + '", "--train_split=', + train_split.uri, + '", "--eval_split=', + eval_split.uri, + '", "--test_split=', + test_split.uri, + '", "--materialized_train_split=', + materialized_train_split.uri, + '", "--materialized_eval_split=', + materialized_eval_split.uri, + '", "--materialized_test_split=', + materialized_test_split.uri, + '", "--training_schema_path=', + training_schema_uri.uri, + ( + f'", "--job_name=automl-tabular-transform-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}' + ), + '", "--dataflow_project=', + project, + '", "--error_file_path=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/error.pb",' + ' "--dataflow_staging_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_staging",' + ' "--dataflow_tmp_dir=' + ), + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/{dsl.PIPELINE_TASK_ID_PLACEHOLDER}/dataflow_tmp",' + ' "--dataflow_max_num_workers=' + ), + dataflow_max_num_workers, + '", "--dataflow_machine_type=', + dataflow_machine_type, + '", "--dataflow_worker_container_image=', + 'us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20231002_0125', + '", "--dataflow_disk_size_gb=', + dataflow_disk_size_gb, + '", "--dataflow_subnetwork_fully_qualified=', + dataflow_subnetwork, + '", "--dataflow_use_public_ips=', + dataflow_use_public_ips, + '", "--dataflow_kms_key=', + encryption_spec_key_name, + '", "--dataflow_service_account=', + dataflow_service_account, + '", "--lro_job_info=', + root_dir, + ( + f'/{dsl.PIPELINE_JOB_ID_PLACEHOLDER}/lro",' + ' "--gcp_resources_path=' + ), + gcp_resources, + '"]}}]}}', + ] + ), + ], + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/utils.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/utils.py new file mode 100644 index 0000000000..6889db79ae --- /dev/null +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/tabular/utils.py @@ -0,0 +1,1438 @@ +"""Util functions for AutoML Tabular pipeline.""" + +import json +import math +import os +import pathlib +from typing import Any, Dict, List, Optional, Tuple +import warnings + +_DEFAULT_NUM_PARALLEL_TRAILS = 35 +_DEFAULT_STAGE_2_NUM_SELECTED_TRAILS = 5 +_NUM_FOLDS = 5 +_DISTILL_TOTAL_TRIALS = 100 +_EVALUATION_BATCH_PREDICT_MACHINE_TYPE = 'n1-highmem-8' +_EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT = 20 +_EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT = 20 +_EVALUATION_BATCH_EXPLAIN_MACHINE_TYPE = 'n1-highmem-8' +_EVALUATION_BATCH_EXPLAIN_STARTING_REPLICA_COUNT = 10 +_EVALUATION_BATCH_EXPLAIN_MAX_REPLICA_COUNT = 10 +_EVALUATION_DATAFLOW_MACHINE_TYPE = 'n1-standard-4' +_EVALUATION_DATAFLOW_STARTING_NUM_WORKERS = 10 +_EVALUATION_DATAFLOW_MAX_NUM_WORKERS = 100 +_EVALUATION_DATAFLOW_DISK_SIZE_GB = 50 + +# Needed because we reference the AutoML Tabular V2 pipeline. +_GCPC_STAGING_PATH = pathlib.Path( + __file__ +).parent.parent.parent.parent.resolve() +_GCPC_PREVIEW_TABULAR_PATH = ( + _GCPC_STAGING_PATH / 'preview' / 'automl' / 'tabular' +) + + +# TODO(b/277393122): Once we finish L2L+FTE integration, add use_fte flag +# to signify FTE usage instead of the presence of num_selected_features. +def _get_default_pipeline_params( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + transformations: str, + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: Optional[int] = None, + stage_2_num_parallel_trials: Optional[int] = None, + stage_2_num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[float] = None, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + optimization_objective_recall_value: Optional[float] = None, + optimization_objective_precision_value: Optional[float] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: Optional[str] = None, + stats_and_example_gen_dataflow_max_num_workers: Optional[int] = None, + stats_and_example_gen_dataflow_disk_size_gb: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: Optional[str] = None, + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: Optional[str] = None, + max_selected_features: Optional[int] = None, + apply_feature_selection_tuning: bool = False, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_batch_explain_machine_type: Optional[str] = None, + evaluation_batch_explain_starting_replica_count: Optional[int] = None, + evaluation_batch_explain_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, + run_distillation: bool = False, + distill_batch_predict_machine_type: Optional[str] = None, + distill_batch_predict_starting_replica_count: Optional[int] = None, + distill_batch_predict_max_replica_count: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + quantiles: Optional[List[float]] = None, + enable_probabilistic_inference: bool = False, + num_selected_features: Optional[int] = None, + model_display_name: str = '', + model_description: str = '', +) -> Dict[str, Any]: + """Get the AutoML Tabular v1 default training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The path to a GCS file containing the transformations to + apply. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + stratified_split_key: The stratified_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + weight_column: The weight column name. + study_spec_parameters_override: The list for overriding study spec. The list + should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + max_selected_features: number of features to select for training, + apply_feature_selection_tuning: tuning feature selection rate if true. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_batch_explain_machine_type: The prediction server machine type + for batch explain components during evaluation. + evaluation_batch_explain_starting_replica_count: The initial number of + prediction server for batch explain components during evaluation. + evaluation_batch_explain_max_replica_count: The max number of prediction + server for batch explain components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + run_distillation: Whether to run distill in the training pipeline. + distill_batch_predict_machine_type: The prediction server machine type for + batch predict component in the model distillation. + distill_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict component in the model distillation. + distill_batch_predict_max_replica_count: The max number of prediction server + for batch predict component in the model distillation. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + quantiles: Quantiles to use for probabilistic inference. Up to 5 quantiles + are allowed of values between 0 and 1, exclusive. Represents the quantiles + to use for that objective. Quantiles must be unique. + enable_probabilistic_inference: If probabilistic inference is enabled, the + model will fit a distribution that captures the uncertainty of a + prediction. At inference time, the predictive distribution is used to make + a point prediction that minimizes the optimization objective. For example, + the mean of a predictive distribution is the point prediction that + minimizes RMSE loss. If quantiles are specified, then the quantiles of the + distribution are also returned. + num_selected_features: Number of selected features for feature selection, + defaults to None, in which case all features are used. If specified, + enable_probabilistic_inference and run_distillation cannot be enabled. + model_display_name: The display name of the uploaded Vertex model. + model_description: The description for the uploaded model. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + if not study_spec_parameters_override: + study_spec_parameters_override = [] + if not stage_1_tuner_worker_pool_specs_override: + stage_1_tuner_worker_pool_specs_override = [] + if not cv_trainer_worker_pool_specs_override: + cv_trainer_worker_pool_specs_override = [] + if not quantiles: + quantiles = [] + + parameter_values = {} + parameters = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column': target_column, + 'prediction_type': prediction_type, + 'data_source_csv_filenames': data_source_csv_filenames, + 'data_source_bigquery_table_path': data_source_bigquery_table_path, + 'predefined_split_key': predefined_split_key, + 'timestamp_split_key': timestamp_split_key, + 'stratified_split_key': stratified_split_key, + 'training_fraction': training_fraction, + 'validation_fraction': validation_fraction, + 'test_fraction': test_fraction, + 'optimization_objective': optimization_objective, + 'train_budget_milli_node_hours': train_budget_milli_node_hours, + 'stage_1_num_parallel_trials': stage_1_num_parallel_trials, + 'stage_2_num_parallel_trials': stage_2_num_parallel_trials, + 'stage_2_num_selected_trials': stage_2_num_selected_trials, + 'weight_column': weight_column, + 'optimization_objective_recall_value': ( + optimization_objective_recall_value + ), + 'optimization_objective_precision_value': ( + optimization_objective_precision_value + ), + 'study_spec_parameters_override': study_spec_parameters_override, + 'stage_1_tuner_worker_pool_specs_override': ( + stage_1_tuner_worker_pool_specs_override + ), + 'cv_trainer_worker_pool_specs_override': ( + cv_trainer_worker_pool_specs_override + ), + 'export_additional_model_without_custom_ops': ( + export_additional_model_without_custom_ops + ), + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'dataflow_service_account': dataflow_service_account, + 'encryption_spec_key_name': encryption_spec_key_name, + 'max_selected_features': max_selected_features, + 'stage_1_tuning_result_artifact_uri': stage_1_tuning_result_artifact_uri, + 'quantiles': quantiles, + 'enable_probabilistic_inference': enable_probabilistic_inference, + 'model_display_name': model_display_name, + 'model_description': model_description, + } + parameter_values.update( + {param: value for param, value in parameters.items() if value is not None} + ) + + if run_evaluation: + eval_parameters = { + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_batch_explain_machine_type': ( + evaluation_batch_explain_machine_type + ), + 'evaluation_batch_explain_starting_replica_count': ( + evaluation_batch_explain_starting_replica_count + ), + 'evaluation_batch_explain_max_replica_count': ( + evaluation_batch_explain_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_starting_num_workers': ( + evaluation_dataflow_starting_num_workers + ), + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'run_evaluation': run_evaluation, + } + parameter_values.update( + { + param: value + for param, value in eval_parameters.items() + if value is not None + } + ) + + # V1 pipeline without FTE + if num_selected_features is None: + if not additional_experiments: + additional_experiments = {} + + parameters = { + 'transformations': transformations, + 'stats_and_example_gen_dataflow_machine_type': ( + stats_and_example_gen_dataflow_machine_type + ), + 'stats_and_example_gen_dataflow_max_num_workers': ( + stats_and_example_gen_dataflow_max_num_workers + ), + 'stats_and_example_gen_dataflow_disk_size_gb': ( + stats_and_example_gen_dataflow_disk_size_gb + ), + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': ( + transform_dataflow_max_num_workers + ), + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'additional_experiments': additional_experiments, + } + parameter_values.update( + { + param: value + for param, value in parameters.items() + if value is not None + } + ) + + if apply_feature_selection_tuning: + parameter_values.update({ + 'apply_feature_selection_tuning': apply_feature_selection_tuning, + }) + + if run_distillation: + distillation_parameters = { + 'distill_batch_predict_machine_type': ( + distill_batch_predict_machine_type + ), + 'distill_batch_predict_starting_replica_count': ( + distill_batch_predict_starting_replica_count + ), + 'distill_batch_predict_max_replica_count': ( + distill_batch_predict_max_replica_count + ), + 'run_distillation': run_distillation, + } + parameter_values.update( + { + param: value + for param, value in distillation_parameters.items() + if value is not None + } + ) + + # V2 pipeline (with FTE) + else: + if run_distillation: + raise ValueError( + 'Distillation is currently not supported' + ' when num_selected_features is specified.' + ) + + parameters = { + 'num_selected_features': num_selected_features, + 'dataset_level_custom_transformation_definitions': [], + 'dataset_level_transformations': [], + 'tf_auto_transform_features': {}, + 'tf_custom_transformation_definitions': [], + 'legacy_transformations_path': transformations, + 'feature_transform_engine_dataflow_machine_type': ( + transform_dataflow_machine_type + ), + 'feature_transform_engine_dataflow_max_num_workers': ( + transform_dataflow_max_num_workers + ), + 'feature_transform_engine_dataflow_disk_size_gb': ( + transform_dataflow_disk_size_gb + ), + } + parameter_values.update( + { + param: value + for param, value in parameters.items() + if value is not None + } + ) + + return parameter_values + + +def get_automl_tabular_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + transformations: str, + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: Optional[int] = None, + stage_2_num_parallel_trials: Optional[int] = None, + stage_2_num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + study_spec_parameters_override: Optional[List[Dict[str, Any]]] = None, + optimization_objective_recall_value: Optional[float] = None, + optimization_objective_precision_value: Optional[float] = None, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: Optional[str] = None, + stats_and_example_gen_dataflow_max_num_workers: Optional[int] = None, + stats_and_example_gen_dataflow_disk_size_gb: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: Optional[str] = None, + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: Optional[str] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_batch_explain_machine_type: Optional[str] = None, + evaluation_batch_explain_starting_replica_count: Optional[int] = None, + evaluation_batch_explain_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, + run_distillation: bool = False, + distill_batch_predict_machine_type: Optional[str] = None, + distill_batch_predict_starting_replica_count: Optional[int] = None, + distill_batch_predict_max_replica_count: Optional[int] = None, + stage_1_tuning_result_artifact_uri: Optional[str] = None, + quantiles: Optional[List[float]] = None, + enable_probabilistic_inference: bool = False, + num_selected_features: Optional[int] = None, + model_display_name: str = '', + model_description: str = '', +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular v1 default training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The path to a GCS file containing the transformations to + apply. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + stratified_split_key: The stratified_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + weight_column: The weight column name. + study_spec_parameters_override: The list for overriding study spec. The list + should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_batch_explain_machine_type: The prediction server machine type + for batch explain components during evaluation. + evaluation_batch_explain_starting_replica_count: The initial number of + prediction server for batch explain components during evaluation. + evaluation_batch_explain_max_replica_count: The max number of prediction + server for batch explain components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + run_distillation: Whether to run distill in the training pipeline. + distill_batch_predict_machine_type: The prediction server machine type for + batch predict component in the model distillation. + distill_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict component in the model distillation. + distill_batch_predict_max_replica_count: The max number of prediction server + for batch predict component in the model distillation. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + quantiles: Quantiles to use for probabilistic inference. Up to 5 quantiles + are allowed of values between 0 and 1, exclusive. Represents the quantiles + to use for that objective. Quantiles must be unique. + enable_probabilistic_inference: If probabilistic inference is enabled, the + model will fit a distribution that captures the uncertainty of a + prediction. At inference time, the predictive distribution is used to make + a point prediction that minimizes the optimization objective. For example, + the mean of a predictive distribution is the point prediction that + minimizes RMSE loss. If quantiles are specified, then the quantiles of the + distribution are also returned. + num_selected_features: Number of selected features for feature selection, + defaults to None, in which case all features are used. + model_display_name: The display name of the uploaded Vertex model. + model_description: The description for the uploaded model. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + parameter_values = _get_default_pipeline_params( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + prediction_type=prediction_type, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + stage_2_num_selected_trials=stage_2_num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + timestamp_split_key=timestamp_split_key, + stratified_split_key=stratified_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + study_spec_parameters_override=study_spec_parameters_override, + optimization_objective_recall_value=optimization_objective_recall_value, + optimization_objective_precision_value=optimization_objective_precision_value, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + cv_trainer_worker_pool_specs_override=cv_trainer_worker_pool_specs_override, + export_additional_model_without_custom_ops=export_additional_model_without_custom_ops, + stats_and_example_gen_dataflow_machine_type=stats_and_example_gen_dataflow_machine_type, + stats_and_example_gen_dataflow_max_num_workers=stats_and_example_gen_dataflow_max_num_workers, + stats_and_example_gen_dataflow_disk_size_gb=stats_and_example_gen_dataflow_disk_size_gb, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + additional_experiments=additional_experiments, + dataflow_service_account=dataflow_service_account, + run_evaluation=run_evaluation, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_batch_explain_machine_type=evaluation_batch_explain_machine_type, + evaluation_batch_explain_starting_replica_count=evaluation_batch_explain_starting_replica_count, + evaluation_batch_explain_max_replica_count=evaluation_batch_explain_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + run_distillation=run_distillation, + distill_batch_predict_machine_type=distill_batch_predict_machine_type, + distill_batch_predict_starting_replica_count=distill_batch_predict_starting_replica_count, + distill_batch_predict_max_replica_count=distill_batch_predict_max_replica_count, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + quantiles=quantiles, + enable_probabilistic_inference=enable_probabilistic_inference, + num_selected_features=num_selected_features, + model_display_name=model_display_name, + model_description=model_description, + ) + + # V1 pipeline without FTE + if num_selected_features is None: + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), 'automl_tabular_pipeline.yaml' + ) + + # V2 pipeline with FTE + else: + pipeline_definition_path = os.path.join( + _GCPC_PREVIEW_TABULAR_PATH, + 'automl_tabular_v2_pipeline.yaml', + ) + # V2 pipeline requires execution engine to be set. + if 'tf_transform_execution_engine' not in parameter_values: + parameter_values['tf_transform_execution_engine'] = 'dataflow' + + return pipeline_definition_path, parameter_values + + +def input_dictionary_to_parameter(input_dict: Optional[Dict[str, Any]]) -> str: + """Convert json input dict to encoded parameter string. + + This function is required due to the limitation on YAML component definition + that YAML definition does not have a keyword for apply quote escape, so the + JSON argument's quote must be manually escaped using this function. + + Args: + input_dict: The input json dictionary. + + Returns: + The encoded string used for parameter. + """ + if not input_dict: + return '' + out = json.dumps(json.dumps(input_dict)) + return out[1:-1] # remove the outside quotes, e.g., "foo" -> foo + + +def get_skip_evaluation_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column_name: str, + prediction_type: str, + optimization_objective: str, + transformations: Dict[str, Any], + split_spec: Dict[str, Any], + data_source: Dict[str, Any], + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: int = _DEFAULT_NUM_PARALLEL_TRAILS, + stage_2_num_parallel_trials: int = _DEFAULT_NUM_PARALLEL_TRAILS, + stage_2_num_selected_trials: int = _DEFAULT_STAGE_2_NUM_SELECTED_TRAILS, + weight_column_name: str = '', + study_spec_override: Optional[Dict[str, Any]] = None, + optimization_objective_recall_value: float = -1, + optimization_objective_precision_value: float = -1, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: str = 'n1-standard-16', + stats_and_example_gen_dataflow_max_num_workers: int = 25, + stats_and_example_gen_dataflow_disk_size_gb: int = 40, + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + additional_experiments: Optional[Dict[str, Any]] = None, +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular training pipeline that skips evaluation. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column_name: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The transformations to apply. + split_spec: The split spec. + data_source: The data source. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + weight_column_name: The weight column name. + study_spec_override: The dictionary for overriding study spec. The + dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + return get_default_pipeline_and_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column_name=target_column_name, + prediction_type=prediction_type, + optimization_objective=optimization_objective, + transformations=transformations, + split_spec=split_spec, + data_source=data_source, + train_budget_milli_node_hours=train_budget_milli_node_hours, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + stage_2_num_selected_trials=stage_2_num_selected_trials, + weight_column_name=weight_column_name, + study_spec_override=study_spec_override, + optimization_objective_recall_value=optimization_objective_recall_value, + optimization_objective_precision_value=optimization_objective_precision_value, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + cv_trainer_worker_pool_specs_override=cv_trainer_worker_pool_specs_override, + export_additional_model_without_custom_ops=export_additional_model_without_custom_ops, + stats_and_example_gen_dataflow_machine_type=stats_and_example_gen_dataflow_machine_type, + stats_and_example_gen_dataflow_max_num_workers=stats_and_example_gen_dataflow_max_num_workers, + stats_and_example_gen_dataflow_disk_size_gb=stats_and_example_gen_dataflow_disk_size_gb, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + additional_experiments=additional_experiments, + run_evaluation=False, + run_distillation=False, + ) + + +def get_default_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column_name: str, + prediction_type: str, + optimization_objective: str, + transformations: Dict[str, Any], + split_spec: Dict[str, Any], + data_source: Dict[str, Any], + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: int = _DEFAULT_NUM_PARALLEL_TRAILS, + stage_2_num_parallel_trials: int = _DEFAULT_NUM_PARALLEL_TRAILS, + stage_2_num_selected_trials: int = _DEFAULT_STAGE_2_NUM_SELECTED_TRAILS, + weight_column_name: str = '', + study_spec_override: Optional[Dict[str, Any]] = None, + optimization_objective_recall_value: float = -1, + optimization_objective_precision_value: float = -1, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: str = 'n1-standard-16', + stats_and_example_gen_dataflow_max_num_workers: int = 25, + stats_and_example_gen_dataflow_disk_size_gb: int = 40, + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: str = '', + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: str = _EVALUATION_BATCH_PREDICT_MACHINE_TYPE, + evaluation_batch_predict_starting_replica_count: int = _EVALUATION_BATCH_PREDICT_STARTING_REPLICA_COUNT, + evaluation_batch_predict_max_replica_count: int = _EVALUATION_BATCH_PREDICT_MAX_REPLICA_COUNT, + evaluation_dataflow_machine_type: str = _EVALUATION_DATAFLOW_MACHINE_TYPE, + evaluation_dataflow_max_num_workers: int = _EVALUATION_DATAFLOW_MAX_NUM_WORKERS, + evaluation_dataflow_disk_size_gb: int = _EVALUATION_DATAFLOW_DISK_SIZE_GB, + run_distillation: bool = False, + distill_batch_predict_machine_type: str = 'n1-standard-16', + distill_batch_predict_starting_replica_count: int = 25, + distill_batch_predict_max_replica_count: int = 25, +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular default training pipeline. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column_name: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The transformations to apply. + split_spec: The split spec. + data_source: The data source. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + weight_column_name: The weight column name. + study_spec_override: The dictionary for overriding study spec. The + dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + run_distillation: Whether to run distill in the training pipeline. + distill_batch_predict_machine_type: The prediction server machine type for + batch predict component in the model distillation. + distill_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict component in the model distillation. + distill_batch_predict_max_replica_count: The max number of prediction server + for batch predict component in the model distillation. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + warnings.warn( + 'This method is deprecated,' + ' please use get_automl_tabular_pipeline_and_parameters instead.' + ) + + if stage_1_num_parallel_trials <= 0: + stage_1_num_parallel_trials = _DEFAULT_NUM_PARALLEL_TRAILS + + if stage_2_num_parallel_trials <= 0: + stage_2_num_parallel_trials = _DEFAULT_NUM_PARALLEL_TRAILS + + hours = float(train_budget_milli_node_hours) / 1000.0 + multiplier = stage_1_num_parallel_trials * hours / 500.0 + stage_1_single_run_max_secs = int(math.sqrt(multiplier) * 2400.0) + phase_2_rounds = int( + math.sqrt(multiplier) * 100 / stage_2_num_parallel_trials + 0.5 + ) + if phase_2_rounds < 1: + phase_2_rounds = 1 + + # All of magic number "1.3" above is because the trial doesn't always finish + # in time_per_trial. 1.3 is an empirical safety margin here. + stage_1_deadline_secs = int( + hours * 3600.0 - 1.3 * stage_1_single_run_max_secs * phase_2_rounds + ) + + if stage_1_deadline_secs < hours * 3600.0 * 0.5: + stage_1_deadline_secs = int(hours * 3600.0 * 0.5) + # Phase 1 deadline is the same as phase 2 deadline in this case. Phase 2 + # can't finish in time after the deadline is cut, so adjust the time per + # trial to meet the deadline. + stage_1_single_run_max_secs = int( + stage_1_deadline_secs / (1.3 * phase_2_rounds) + ) + + reduce_search_space_mode = 'minimal' + if multiplier > 2: + reduce_search_space_mode = 'regular' + if multiplier > 4: + reduce_search_space_mode = 'full' + + # Stage 2 number of trials is stage_1_num_selected_trials * + # _NUM_FOLDS, which should be equal to phase_2_rounds * + # stage_2_num_parallel_trials. Use this information to calculate + # stage_1_num_selected_trials: + stage_1_num_selected_trials = int( + phase_2_rounds * stage_2_num_parallel_trials / _NUM_FOLDS + ) + stage_1_deadline_hours = stage_1_deadline_secs / 3600.0 + + stage_2_deadline_hours = hours - stage_1_deadline_hours + stage_2_single_run_max_secs = stage_1_single_run_max_secs + + parameter_values = { + 'project': project, + 'location': location, + 'root_dir': root_dir, + 'target_column_name': target_column_name, + 'prediction_type': prediction_type, + 'optimization_objective': optimization_objective, + 'transformations': input_dictionary_to_parameter(transformations), + 'split_spec': input_dictionary_to_parameter(split_spec), + 'data_source': input_dictionary_to_parameter(data_source), + 'stage_1_deadline_hours': stage_1_deadline_hours, + 'stage_1_num_parallel_trials': stage_1_num_parallel_trials, + 'stage_1_num_selected_trials': stage_1_num_selected_trials, + 'stage_1_single_run_max_secs': stage_1_single_run_max_secs, + 'reduce_search_space_mode': reduce_search_space_mode, + 'stage_2_deadline_hours': stage_2_deadline_hours, + 'stage_2_num_parallel_trials': stage_2_num_parallel_trials, + 'stage_2_num_selected_trials': stage_2_num_selected_trials, + 'stage_2_single_run_max_secs': stage_2_single_run_max_secs, + 'weight_column_name': weight_column_name, + 'optimization_objective_recall_value': ( + optimization_objective_recall_value + ), + 'optimization_objective_precision_value': ( + optimization_objective_precision_value + ), + 'study_spec_override': input_dictionary_to_parameter(study_spec_override), + 'stage_1_tuner_worker_pool_specs_override': input_dictionary_to_parameter( + stage_1_tuner_worker_pool_specs_override + ), + 'cv_trainer_worker_pool_specs_override': input_dictionary_to_parameter( + cv_trainer_worker_pool_specs_override + ), + 'export_additional_model_without_custom_ops': ( + export_additional_model_without_custom_ops + ), + 'stats_and_example_gen_dataflow_machine_type': ( + stats_and_example_gen_dataflow_machine_type + ), + 'stats_and_example_gen_dataflow_max_num_workers': ( + stats_and_example_gen_dataflow_max_num_workers + ), + 'stats_and_example_gen_dataflow_disk_size_gb': ( + stats_and_example_gen_dataflow_disk_size_gb + ), + 'transform_dataflow_machine_type': transform_dataflow_machine_type, + 'transform_dataflow_max_num_workers': transform_dataflow_max_num_workers, + 'transform_dataflow_disk_size_gb': transform_dataflow_disk_size_gb, + 'dataflow_subnetwork': dataflow_subnetwork, + 'dataflow_use_public_ips': dataflow_use_public_ips, + 'encryption_spec_key_name': encryption_spec_key_name, + } + if additional_experiments: + parameter_values.update( + { + 'additional_experiments': input_dictionary_to_parameter( + additional_experiments + ) + } + ) + if run_evaluation: + parameter_values.update({ + 'dataflow_service_account': dataflow_service_account, + 'evaluation_batch_predict_machine_type': ( + evaluation_batch_predict_machine_type + ), + 'evaluation_batch_predict_starting_replica_count': ( + evaluation_batch_predict_starting_replica_count + ), + 'evaluation_batch_predict_max_replica_count': ( + evaluation_batch_predict_max_replica_count + ), + 'evaluation_dataflow_machine_type': evaluation_dataflow_machine_type, + 'evaluation_dataflow_max_num_workers': ( + evaluation_dataflow_max_num_workers + ), + 'evaluation_dataflow_disk_size_gb': evaluation_dataflow_disk_size_gb, + 'run_evaluation': run_evaluation, + }) + if run_distillation: + # All of magic number "1.3" above is because the trial doesn't always finish + # in time_per_trial. 1.3 is an empirical safety margin here. + distill_stage_1_deadline_hours = ( + math.ceil( + float(_DISTILL_TOTAL_TRIALS) + / parameter_values['stage_1_num_parallel_trials'] + ) + * parameter_values['stage_1_single_run_max_secs'] + * 1.3 + / 3600.0 + ) + + parameter_values.update({ + 'distill_stage_1_deadline_hours': distill_stage_1_deadline_hours, + 'distill_batch_predict_machine_type': ( + distill_batch_predict_machine_type + ), + 'distill_batch_predict_starting_replica_count': ( + distill_batch_predict_starting_replica_count + ), + 'distill_batch_predict_max_replica_count': ( + distill_batch_predict_max_replica_count + ), + 'run_distillation': run_distillation, + }) + pipeline_definition_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), + 'deprecated/default_pipeline.json', + ) + return pipeline_definition_path, parameter_values + + +def get_skip_architecture_search_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column: str, + prediction_type: str, + optimization_objective: str, + transformations: str, + train_budget_milli_node_hours: float, + stage_1_tuning_result_artifact_uri: str, + stage_2_num_parallel_trials: Optional[int] = None, + stage_2_num_selected_trials: Optional[int] = None, + data_source_csv_filenames: Optional[str] = None, + data_source_bigquery_table_path: Optional[str] = None, + predefined_split_key: Optional[str] = None, + timestamp_split_key: Optional[str] = None, + stratified_split_key: Optional[str] = None, + training_fraction: Optional[float] = None, + validation_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + weight_column: Optional[str] = None, + optimization_objective_recall_value: Optional[float] = None, + optimization_objective_precision_value: Optional[float] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: Optional[str] = None, + stats_and_example_gen_dataflow_max_num_workers: Optional[int] = None, + stats_and_example_gen_dataflow_disk_size_gb: Optional[int] = None, + transform_dataflow_machine_type: Optional[str] = None, + transform_dataflow_max_num_workers: Optional[int] = None, + transform_dataflow_disk_size_gb: Optional[int] = None, + dataflow_subnetwork: Optional[str] = None, + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: Optional[str] = None, + additional_experiments: Optional[Dict[str, Any]] = None, + dataflow_service_account: Optional[str] = None, + run_evaluation: bool = True, + evaluation_batch_predict_machine_type: Optional[str] = None, + evaluation_batch_predict_starting_replica_count: Optional[int] = None, + evaluation_batch_predict_max_replica_count: Optional[int] = None, + evaluation_batch_explain_machine_type: Optional[str] = None, + evaluation_batch_explain_starting_replica_count: Optional[int] = None, + evaluation_batch_explain_max_replica_count: Optional[int] = None, + evaluation_dataflow_machine_type: Optional[str] = None, + evaluation_dataflow_starting_num_workers: Optional[int] = None, + evaluation_dataflow_max_num_workers: Optional[int] = None, + evaluation_dataflow_disk_size_gb: Optional[int] = None, +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular training pipeline that skips architecture search. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The transformations to apply. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_tuning_result_artifact_uri: The stage 1 tuning result artifact GCS + URI. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + data_source_csv_filenames: The CSV data source. + data_source_bigquery_table_path: The BigQuery data source. + predefined_split_key: The predefined_split column name. + timestamp_split_key: The timestamp_split column name. + stratified_split_key: The stratified_split column name. + training_fraction: The training fraction. + validation_fraction: The validation fraction. + test_fraction: float = The test fraction. + weight_column: The weight column name. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + dataflow_service_account: Custom service account to run dataflow jobs. + run_evaluation: Whether to run evaluation in the training pipeline. + evaluation_batch_predict_machine_type: The prediction server machine type + for batch predict components during evaluation. + evaluation_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict components during evaluation. + evaluation_batch_predict_max_replica_count: The max number of prediction + server for batch predict components during evaluation. + evaluation_batch_explain_machine_type: The prediction server machine type + for batch explain components during evaluation. + evaluation_batch_explain_starting_replica_count: The initial number of + prediction server for batch explain components during evaluation. + evaluation_batch_explain_max_replica_count: The max number of prediction + server for batch explain components during evaluation. + evaluation_dataflow_machine_type: The dataflow machine type for evaluation + components. + evaluation_dataflow_starting_num_workers: The initial number of Dataflow + workers for evaluation components. + evaluation_dataflow_max_num_workers: The max number of Dataflow workers for + evaluation components. + evaluation_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + evaluation components. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + + return get_automl_tabular_pipeline_and_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column=target_column, + prediction_type=prediction_type, + optimization_objective=optimization_objective, + transformations=transformations, + train_budget_milli_node_hours=train_budget_milli_node_hours, + stage_1_num_parallel_trials=None, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + stage_2_num_selected_trials=stage_2_num_selected_trials, + data_source_csv_filenames=data_source_csv_filenames, + data_source_bigquery_table_path=data_source_bigquery_table_path, + predefined_split_key=predefined_split_key, + timestamp_split_key=timestamp_split_key, + stratified_split_key=stratified_split_key, + training_fraction=training_fraction, + validation_fraction=validation_fraction, + test_fraction=test_fraction, + weight_column=weight_column, + study_spec_parameters_override=[], + optimization_objective_recall_value=optimization_objective_recall_value, + optimization_objective_precision_value=optimization_objective_precision_value, + stage_1_tuner_worker_pool_specs_override={}, + cv_trainer_worker_pool_specs_override=cv_trainer_worker_pool_specs_override, + export_additional_model_without_custom_ops=export_additional_model_without_custom_ops, + stats_and_example_gen_dataflow_machine_type=stats_and_example_gen_dataflow_machine_type, + stats_and_example_gen_dataflow_max_num_workers=stats_and_example_gen_dataflow_max_num_workers, + stats_and_example_gen_dataflow_disk_size_gb=stats_and_example_gen_dataflow_disk_size_gb, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + additional_experiments=additional_experiments, + dataflow_service_account=dataflow_service_account, + run_evaluation=run_evaluation, + evaluation_batch_predict_machine_type=evaluation_batch_predict_machine_type, + evaluation_batch_predict_starting_replica_count=evaluation_batch_predict_starting_replica_count, + evaluation_batch_predict_max_replica_count=evaluation_batch_predict_max_replica_count, + evaluation_batch_explain_machine_type=evaluation_batch_explain_machine_type, + evaluation_batch_explain_starting_replica_count=evaluation_batch_explain_starting_replica_count, + evaluation_batch_explain_max_replica_count=evaluation_batch_explain_max_replica_count, + evaluation_dataflow_machine_type=evaluation_dataflow_machine_type, + evaluation_dataflow_starting_num_workers=evaluation_dataflow_starting_num_workers, + evaluation_dataflow_max_num_workers=evaluation_dataflow_max_num_workers, + evaluation_dataflow_disk_size_gb=evaluation_dataflow_disk_size_gb, + run_distillation=None, + distill_batch_predict_machine_type=None, + distill_batch_predict_starting_replica_count=None, + distill_batch_predict_max_replica_count=None, + stage_1_tuning_result_artifact_uri=stage_1_tuning_result_artifact_uri, + quantiles=[], + enable_probabilistic_inference=False, + ) + + +def get_distill_skip_evaluation_pipeline_and_parameters( + project: str, + location: str, + root_dir: str, + target_column_name: str, + prediction_type: str, + optimization_objective: str, + transformations: Dict[str, Any], + split_spec: Dict[str, Any], + data_source: Dict[str, Any], + train_budget_milli_node_hours: float, + stage_1_num_parallel_trials: int = _DEFAULT_NUM_PARALLEL_TRAILS, + stage_2_num_parallel_trials: int = _DEFAULT_NUM_PARALLEL_TRAILS, + stage_2_num_selected_trials: int = _DEFAULT_STAGE_2_NUM_SELECTED_TRAILS, + weight_column_name: str = '', + study_spec_override: Optional[Dict[str, Any]] = None, + optimization_objective_recall_value: float = -1, + optimization_objective_precision_value: float = -1, + stage_1_tuner_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + cv_trainer_worker_pool_specs_override: Optional[Dict[str, Any]] = None, + export_additional_model_without_custom_ops: bool = False, + stats_and_example_gen_dataflow_machine_type: str = 'n1-standard-16', + stats_and_example_gen_dataflow_max_num_workers: int = 25, + stats_and_example_gen_dataflow_disk_size_gb: int = 40, + transform_dataflow_machine_type: str = 'n1-standard-16', + transform_dataflow_max_num_workers: int = 25, + transform_dataflow_disk_size_gb: int = 40, + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + additional_experiments: Optional[Dict[str, Any]] = None, + distill_batch_predict_machine_type: str = 'n1-standard-16', + distill_batch_predict_starting_replica_count: int = 25, + distill_batch_predict_max_replica_count: int = 25, +) -> Tuple[str, Dict[str, Any]]: + """Get the AutoML Tabular training pipeline that distill and skips evaluation. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + root_dir: The root GCS directory for the pipeline components. + target_column_name: The target column name. + prediction_type: The type of prediction the model is to produce. + "classification" or "regression". + optimization_objective: For binary classification, "maximize-au-roc", + "minimize-log-loss", "maximize-au-prc", "maximize-precision-at-recall", or + "maximize-recall-at-precision". For multi class classification, + "minimize-log-loss". For regression, "minimize-rmse", "minimize-mae", or + "minimize-rmsle". + transformations: The transformations to apply. + split_spec: The split spec. + data_source: The data source. + train_budget_milli_node_hours: The train budget of creating this model, + expressed in milli node hours i.e. 1,000 value in this field means 1 node + hour. + stage_1_num_parallel_trials: Number of parallel trails for stage 1. + stage_2_num_parallel_trials: Number of parallel trails for stage 2. + stage_2_num_selected_trials: Number of selected trials for stage 2. + weight_column_name: The weight column name. + study_spec_override: The dictionary for overriding study spec. The + dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/study.proto#L181. + optimization_objective_recall_value: Required when optimization_objective is + "maximize-precision-at-recall". Must be between 0 and 1, inclusive. + optimization_objective_precision_value: Required when optimization_objective + is "maximize-recall-at-precision". Must be between 0 and 1, inclusive. + stage_1_tuner_worker_pool_specs_override: The dictionary for overriding. + stage 1 tuner worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + cv_trainer_worker_pool_specs_override: The dictionary for overriding stage + cv trainer worker pool spec. The dictionary should be of format + https://github.com/googleapis/googleapis/blob/4e836c7c257e3e20b1de14d470993a2b1f4736a8/google/cloud/aiplatform/v1beta1/custom_job.proto#L172. + export_additional_model_without_custom_ops: Whether to export additional + model without custom TensorFlow operators. + stats_and_example_gen_dataflow_machine_type: The dataflow machine type for + stats_and_example_gen component. + stats_and_example_gen_dataflow_max_num_workers: The max number of Dataflow + workers for stats_and_example_gen component. + stats_and_example_gen_dataflow_disk_size_gb: Dataflow worker's disk size in + GB for stats_and_example_gen component. + transform_dataflow_machine_type: The dataflow machine type for transform + component. + transform_dataflow_max_num_workers: The max number of Dataflow workers for + transform component. + transform_dataflow_disk_size_gb: Dataflow worker's disk size in GB for + transform component. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty + the default subnetwork will be used. Example: + https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP + addresses. + encryption_spec_key_name: The KMS key name. + additional_experiments: Use this field to config private preview features. + distill_batch_predict_machine_type: The prediction server machine type for + batch predict component in the model distillation. + distill_batch_predict_starting_replica_count: The initial number of + prediction server for batch predict component in the model distillation. + distill_batch_predict_max_replica_count: The max number of prediction server + for batch predict component in the model distillation. + + Returns: + Tuple of pipeline_definition_path and parameter_values. + """ + warnings.warn( + 'Depreciated. Please use get_automl_tabular_pipeline_and_parameters.' + ) + + return get_default_pipeline_and_parameters( + project=project, + location=location, + root_dir=root_dir, + target_column_name=target_column_name, + prediction_type=prediction_type, + optimization_objective=optimization_objective, + transformations=transformations, + split_spec=split_spec, + data_source=data_source, + train_budget_milli_node_hours=train_budget_milli_node_hours, + stage_1_num_parallel_trials=stage_1_num_parallel_trials, + stage_2_num_parallel_trials=stage_2_num_parallel_trials, + stage_2_num_selected_trials=stage_2_num_selected_trials, + weight_column_name=weight_column_name, + study_spec_override=study_spec_override, + optimization_objective_recall_value=optimization_objective_recall_value, + optimization_objective_precision_value=optimization_objective_precision_value, + stage_1_tuner_worker_pool_specs_override=stage_1_tuner_worker_pool_specs_override, + cv_trainer_worker_pool_specs_override=cv_trainer_worker_pool_specs_override, + export_additional_model_without_custom_ops=export_additional_model_without_custom_ops, + stats_and_example_gen_dataflow_machine_type=stats_and_example_gen_dataflow_machine_type, + stats_and_example_gen_dataflow_max_num_workers=stats_and_example_gen_dataflow_max_num_workers, + stats_and_example_gen_dataflow_disk_size_gb=stats_and_example_gen_dataflow_disk_size_gb, + transform_dataflow_machine_type=transform_dataflow_machine_type, + transform_dataflow_max_num_workers=transform_dataflow_max_num_workers, + transform_dataflow_disk_size_gb=transform_dataflow_disk_size_gb, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + additional_experiments=additional_experiments, + distill_batch_predict_machine_type=distill_batch_predict_machine_type, + distill_batch_predict_starting_replica_count=distill_batch_predict_starting_replica_count, + distill_batch_predict_max_replica_count=distill_batch_predict_max_replica_count, + run_evaluation=False, + run_distillation=True, + ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/__init__.py index 73fff4032e..fa2f7099f1 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/__init__.py @@ -11,8 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Create `Vertex AI AutoML training jobs `_ for image, text, video, and forecasting.""" - +# fmt: off +"""Create [Vertex AI AutoML training jobs](https://cloud.google.com/vertex-ai/docs/beginner/beginners-guide) for image, text, video, and forecasting.""" +# fmt: on from google_cloud_pipeline_components.v1.automl.training_job.automl_forecasting_training_job.component import automl_forecasting_training_job as AutoMLForecastingTrainingJobRunOp from google_cloud_pipeline_components.v1.automl.training_job.automl_image_training_job.component import automl_image_training_job as AutoMLImageTrainingJobRunOp from google_cloud_pipeline_components.v1.automl.training_job.automl_tabular_training_job.component import automl_tabular_training_job as AutoMLTabularTrainingJobRunOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_forecasting_training_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_forecasting_training_job/component.py index 003a672ac4..e2466fc76f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_forecasting_training_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_forecasting_training_job/component.py @@ -78,260 +78,65 @@ def automl_forecasting_training_job( # fmt: off """Runs the training job and returns a model. - If training on a Vertex AI dataset, you can use one of the following split configurations: - - Data fraction splits: - Any of ``training_fraction_split``, ``validation_fraction_split`` and - ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If - the provided ones sum to less than 1, the remainder is assigned to sets as - decided by Vertex AI. If none of the fractions are set, by default roughly 80% - of data will be used for training, 10% for validation, and 10% for test. - Predefined splits: - Assigns input data to training, validation, and test sets based on the value of a provided key. - If using predefined splits, ``predefined_split_column_name`` must be provided. - Supported only for tabular Datasets. - Timestamp splits: - Assigns input data to training, validation, and test sets - based on a provided timestamps. The youngest data pieces are - assigned to training set, next to validation set, and the oldest - to the test set. - - Supported only for tabular Datasets. + If training on a Vertex AI dataset, you can use one of the following split configurations: Data fraction splits: Any of `training_fraction_split`, `validation_fraction_split` and `test_fraction_split` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Predefined splits: Assigns input data to training, validation, and test sets based on the value of a provided key. If using predefined splits, `predefined_split_column_name` must be provided. Supported only for tabular Datasets. Timestamp splits: Assigns input data to training, validation, and test sets based on a provided timestamps. The youngest data pieces are assigned to training set, next to validation set, and the oldest to the test set. Supported only for tabular Datasets. Args: - dataset: The dataset within the same Project from which data will be used to train the Model. The - Dataset must use schema compatible with Model being trained, - and what is compatible should be described in the used - TrainingPipeline's [training_task_definition] - [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. - For time series Datasets, all their data is exported to - training, to pick and choose from. - target_column: Name of the column that the Model is to predict values for. This - column must be unavailable at forecast. - time_column: Name of the column that identifies time order in the time series. - This column must be available at forecast. + dataset: The dataset within the same Project from which data will be used to train the Model. The Dataset must use schema compatible with Model being trained, and what is compatible should be described in the used TrainingPipeline's [training_task_definition] [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. For time series Datasets, all their data is exported to training, to pick and choose from. + target_column: Name of the column that the Model is to predict values for. This column must be unavailable at forecast. + time_column: Name of the column that identifies time order in the time series. This column must be available at forecast. time_series_identifier_column: Name of the column that identifies the time series. - unavailable_at_forecast_columns: Column names of columns that are unavailable at forecast. - Each column contains information for the given entity (identified by the - [time_series_identifier_column]) that is unknown before the forecast - (e.g. population of a city in a given year, or weather on a given day). - available_at_forecast_columns: Column names of columns that are available at forecast. - Each column contains information for the given entity (identified by the - [time_series_identifier_column]) that is known at forecast. - forecast_horizon: - The amount of time into the future for which forecasted values for the target are - returned. Expressed in number of units defined by the [data_granularity_unit] and - [data_granularity_count] field. Inclusive. - data_granularity_unit: The data granularity unit. Accepted values are ``minute``, - ``hour``, ``day``, ``week``, ``month``, ``year``. - data_granularity_count: The number of data granularity units between data points in the training - data. If [data_granularity_unit] is `minute`, can be 1, 5, 10, 15, or 30. For all other - values of [data_granularity_unit], must be 1. - training_fraction_split: The fraction of the input data that is to be used to train - the Model. This is ignored if Dataset is not provided. - validation_fraction_split: The fraction of the input data that is to be used to validate - the Model. This is ignored if Dataset is not provided. - test_fraction_split: The fraction of the input data that is to be used to evaluate - the Model. This is ignored if Dataset is not provided. - predefined_split_column_name: The key is a name of one of the Dataset's data - columns. The value of the key (either the label's value or - value in the column) must be one of {``TRAIN``, - ``VALIDATE``, ``TEST``}, and it defines to which set the - given piece of data is assigned. If for a piece of data the - key is not present or has an invalid value, that piece is - ignored by the pipeline. - Supported only for tabular and time series Datasets. - timestamp_split_column_name: The key is a name of one of the Dataset's data - columns. The value of the key values of the key (the values in - the column) must be in RFC 3339 `date-time` format, where - `time-offset` = `"Z"` (e.g. 1985-04-12T23:20:50.52Z). If for a - piece of data the key is not present or has an invalid value, - that piece is ignored by the pipeline. - Supported only for tabular and time series Datasets. - This parameter must be used with training_fraction_split, - validation_fraction_split, and test_fraction_split. - weight_column: Name of the column that should be used as the weight column. - Higher values in this column give more importance to the row - during Model training. The column must have numeric values between 0 and - 10000 inclusively, and 0 value means that the row is ignored. - If the weight column field is not set, then all rows are assumed to have - equal weight of 1. - time_series_attribute_columns: Column names that should be used as attribute columns. - Each column is constant within a time series. - context_window: The amount of time into the past training and prediction data is used for - model training and prediction respectively. Expressed in number of units defined by the - [data_granularity_unit] and [data_granularity_count] fields. When not provided uses the - default value of 0 which means the model sets each series context window to be 0 (also - known as "cold start"). Inclusive. - export_evaluated_data_items: Whether to export the test set predictions to a BigQuery table. - If False, then the export is not performed. - export_evaluated_data_items_bigquery_destination_uri: URI of desired destination BigQuery table for exported test set predictions. - Expected format: - ``bq://::`` - If not specified, then results are exported to the following auto-created BigQuery - table: - ``:export_evaluated_examples__.evaluated_examples`` - Applies only if [export_evaluated_data_items] is True. - export_evaluated_data_items_override_destination: Whether to override the contents of [export_evaluated_data_items_bigquery_destination_uri], - if the table exists, for exported test set predictions. If False, and the - table exists, then the training job will fail. - Applies only if [export_evaluated_data_items] is True and - [export_evaluated_data_items_bigquery_destination_uri] is specified. - quantiles: Quantiles to use for the `minimize-quantile-loss` - [AutoMLForecastingTrainingJob.optimization_objective]. This argument is required in - this case. - Accepts up to 5 quantiles in the form of a double from 0 to 1, exclusive. - Each quantile must be unique. - validation_options: Validation options for the data validation component. The available options are: - "fail-pipeline" - (default), will validate against the validation and fail the pipeline if it fails. - "ignore-validation" - ignore the results of the validation and continue the pipeline - budget_milli_node_hours: The train budget of creating this Model, expressed in milli node - hours i.e. 1,000 value in this field means 1 node hour. - The training cost of the model will not exceed this budget. The final - cost will be attempted to be close to the budget, though may end up - being (even) noticeably smaller - at the backend's discretion. This - especially may happen when further model training ceases to provide - any improvements. - If the budget is set to a value known to be insufficient to train a - Model for the given training set, the training won't be attempted and - will error. - The minimum value is 1000 and the maximum is 72000. - model_display_name: If the script produces a managed Vertex AI Model. The display name of - the Model. The name can be up to 128 characters long and can be consist - of any UTF-8 characters. - If not provided upon creation, the job's display_name is used. - model_labels: The labels with user-defined metadata to - organize your Models. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - model_id: The ID to use for the Model produced by this job, - which will become the final component of the model resource name. - This value may be up to 63 characters, and valid characters - are `[a-z0-9_-]`. The first character cannot be a number or hyphen. - parent_model: The resource name or model ID of an existing model. - The new model uploaded by this job will be a version of `parent_model`. - Only set this field when training a new version of an existing model. - is_default_version: When set to True, the newly uploaded model version will - automatically have alias "default" included. Subsequent uses of - the model produced by this job without a version specified will - use this "default" version. - When set to False, the "default" alias will not be moved. - Actions targeting the model version produced by this job will need - to specifically reference this version by ID or alias. - New model uploads, i.e. version 1, will always be "default" aliased. - model_version_aliases: User provided version aliases so that the model version - uploaded by this job can be referenced via alias instead of - auto-generated version ID. A default version alias will be created - for the first version of the model. - The format is [a-z][a-zA-Z0-9-]{0,126}[a-z0-9] + unavailable_at_forecast_columns: Column names of columns that are unavailable at forecast. Each column contains information for the given entity (identified by the [time_series_identifier_column]) that is unknown before the forecast (e.g. population of a city in a given year, or weather on a given day). + available_at_forecast_columns: Column names of columns that are available at forecast. Each column contains information for the given entity (identified by the [time_series_identifier_column]) that is known at forecast. + forecast_horizon: The amount of time into the future for which forecasted values for the target are returned. Expressed in number of units defined by the [data_granularity_unit] and [data_granularity_count] field. Inclusive. + data_granularity_unit: The data granularity unit. Accepted values are `minute`, `hour`, `day`, `week`, `month`, `year`. + data_granularity_count: The number of data granularity units between data points in the training data. If [data_granularity_unit] is `minute`, can be 1, 5, 10, 15, or 30. For all other values of [data_granularity_unit], must be 1. + training_fraction_split: The fraction of the input data that is to be used to train the Model. This is ignored if Dataset is not provided. + validation_fraction_split: The fraction of the input data that is to be used to validate the Model. This is ignored if Dataset is not provided. + test_fraction_split: The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. + predefined_split_column_name: The key is a name of one of the Dataset's data columns. The value of the key (either the label's value or value in the column) must be one of {`TRAIN`, `VALIDATE`, `TEST`}, and it defines to which set the given piece of data is assigned. If for a piece of data the key is not present or has an invalid value, that piece is ignored by the pipeline. Supported only for tabular and time series Datasets. + timestamp_split_column_name: The key is a name of one of the Dataset's data columns. The value of the key values of the key (the values in the column) must be in RFC 3339 `date-time` format, where `time-offset` = `"Z"` (e.g. 1985-04-12T23:20:50.52Z). If for a piece of data the key is not present or has an invalid value, that piece is ignored by the pipeline. Supported only for tabular and time series Datasets. This parameter must be used with training_fraction_split, validation_fraction_split, and test_fraction_split. + weight_column: Name of the column that should be used as the weight column. Higher values in this column give more importance to the row during Model training. The column must have numeric values between 0 and 10000 inclusively, and 0 value means that the row is ignored. If the weight column field is not set, then all rows are assumed to have equal weight of 1. + time_series_attribute_columns: Column names that should be used as attribute columns. Each column is constant within a time series. + context_window: The amount of time into the past training and prediction data is used for model training and prediction respectively. Expressed in number of units defined by the [data_granularity_unit] and [data_granularity_count] fields. When not provided uses the default value of 0 which means the model sets each series context window to be 0 (also known as "cold start"). Inclusive. + export_evaluated_data_items: Whether to export the test set predictions to a BigQuery table. If False, then the export is not performed. + export_evaluated_data_items_bigquery_destination_uri: URI of desired destination BigQuery table for exported test set predictions. Expected format: `bq://::
` If not specified, then results are exported to the following auto-created BigQuery table: `:export_evaluated_examples__.evaluated_examples` Applies only if [export_evaluated_data_items] is True. + export_evaluated_data_items_override_destination: Whether to override the contents of [export_evaluated_data_items_bigquery_destination_uri], if the table exists, for exported test set predictions. If False, and the table exists, then the training job will fail. Applies only if [export_evaluated_data_items] is True and [export_evaluated_data_items_bigquery_destination_uri] is specified. + quantiles: Quantiles to use for the `minimize-quantile-loss` [AutoMLForecastingTrainingJob.optimization_objective]. This argument is required in this case. Accepts up to 5 quantiles in the form of a double from 0 to 1, exclusive. Each quantile must be unique. + validation_options: Validation options for the data validation component. The available options are: "fail-pipeline" - (default), will validate against the validation and fail the pipeline if it fails. "ignore-validation" - ignore the results of the validation and continue the pipeline + budget_milli_node_hours: The train budget of creating this Model, expressed in milli node hours i.e. 1,000 value in this field means 1 node hour. The training cost of the model will not exceed this budget. The final cost will be attempted to be close to the budget, though may end up being (even) noticeably smaller - at the backend's discretion. This especially may happen when further model training ceases to provide any improvements. If the budget is set to a value known to be insufficient to train a Model for the given training set, the training won't be attempted and will error. The minimum value is 1000 and the maximum is 72000. + model_display_name: If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If not provided upon creation, the job's display_name is used. + model_labels: The labels with user-defined metadata to organize your Models. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + model_id: The ID to use for the Model produced by this job, which will become the final component of the model resource name. This value may be up to 63 characters, and valid characters are `[a-z0-9_-]`. The first character cannot be a number or hyphen. + parent_model: The resource name or model ID of an existing model. The new model uploaded by this job will be a version of `parent_model`. Only set this field when training a new version of an existing model. + is_default_version: When set to True, the newly uploaded model version will automatically have alias "default" included. Subsequent uses of the model produced by this job without a version specified will use this "default" version. When set to False, the "default" alias will not be moved. Actions targeting the model version produced by this job will need to specifically reference this version by ID or alias. New model uploads, i.e. version 1, will always be "default" aliased. + model_version_aliases: User provided version aliases so that the model version uploaded by this job can be referenced via alias instead of auto-generated version ID. A default version alias will be created for the first version of the model. The format is [a-z][a-zA-Z0-9-]{0,126}[a-z0-9] model_version_description: The description of the model version being uploaded by this job. - hierarchy_group_columns: A list of time series attribute column names that - define the time series hierarchy. Only one level of hierarchy is - supported, ex. ``region`` for a hierarchy of stores or - ``department`` for a hierarchy of products. If multiple columns - are specified, time series will be grouped by their combined - values, ex. (``blue``, ``large``) for ``color`` and ``size``, up - to 5 columns are accepted. If no group columns are specified, - all time series are considered to be part of the same group. - hierarchy_group_total_weight: The weight of the loss for predictions aggregated over - time series in the same hierarchy group. - hierarchy_temporal_total_weight: The weight of the loss for predictions aggregated over - the horizon for a single time series. - hierarchy_group_temporal_total_weight: The weight of the loss for predictions aggregated over - both the horizon and time series in the same hierarchy group. - window_column: Name of the column that should be used to filter input - rows. The column should contain either booleans or string - booleans; if the value of the row is True, generate a sliding - window from that row. - window_stride_length: Step length used to generate input examples. Every - ``window_stride_length`` rows will be used to generate a sliding - window. - window_max_count: Number of rows that should be used to generate input - examples. If the total row count is larger than this number, the - input data will be randomly sampled to hit the count. - holiday_regions: The geographical regions to use when creating holiday - features. This option is only allowed when data_granularity_unit - is ``day``. Acceptable values can come from any of the following - levels: - Top level: GLOBAL - Second level: continental regions - NA: North America - JAPAC: Japan and Asia Pacific - EMEA: Europe, the Middle East and Africa - LAC: Latin America and the Caribbean - Third level: countries from ISO 3166-1 Country codes. + hierarchy_group_columns: A list of time series attribute column names that define the time series hierarchy. Only one level of hierarchy is supported, ex. `region` for a hierarchy of stores or `department` for a hierarchy of products. If multiple columns are specified, time series will be grouped by their combined values, ex. (`blue`, `large`) for `color` and `size`, up to 5 columns are accepted. If no group columns are specified, all time series are considered to be part of the same group. + hierarchy_group_total_weight: The weight of the loss for predictions aggregated over time series in the same hierarchy group. + hierarchy_temporal_total_weight: The weight of the loss for predictions aggregated over the horizon for a single time series. + hierarchy_group_temporal_total_weight: The weight of the loss for predictions aggregated over both the horizon and time series in the same hierarchy group. + window_column: Name of the column that should be used to filter input rows. The column should contain either booleans or string booleans; if the value of the row is True, generate a sliding window from that row. + window_stride_length: Step length used to generate input examples. Every `window_stride_length` rows will be used to generate a sliding window. + window_max_count: Number of rows that should be used to generate input examples. If the total row count is larger than this number, the input data will be randomly sampled to hit the count. + holiday_regions: The geographical regions to use when creating holiday features. This option is only allowed when data_granularity_unit is `day`. Acceptable values can come from any of the following levels: + Top level: GLOBAL Second level: continental regions + NA: North America + JAPAC: Japan and Asia Pacific + EMEA: Europe, the Middle East and Africa + LAC: Latin America and the Caribbean Third level: countries from ISO 3166-1 Country codes. display_name: The user-defined name of this TrainingPipeline. - optimization_objective: Objective function the model is to be optimized towards. - The training process creates a Model that optimizes the value of the objective - function over the validation set. The supported optimization objectives: - "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). - "minimize-mae" - Minimize mean-absolute error (MAE). - "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). - "minimize-rmspe" - Minimize root-mean-squared percentage error (RMSPE). - "minimize-wape-mae" - Minimize the combination of weighted absolute percentage error (WAPE) and mean-absolute-error (MAE). - "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. - (Set this objective to build quantile forecasts.) - column_specs: Alternative to column_transformations where the keys of the dict - are column names and their respective values are one of - AutoMLTabularTrainingJob.column_data_types. - When creating transformation for BigQuery Struct column, the column - should be flattened using "." as the delimiter. Only columns with no child - should have a transformation. - If an input column has no transformations on it, such a column is - ignored by the training, except for the targetColumn, which should have - no transformations defined on. - Only one of column_transformations or column_specs should be passed. - column_transformations: - Transformations to apply to the input columns (i.e. columns other - than the targetColumn). Each transformation may produce multiple - result values from the column's value, and all are used for training. - When creating transformation for BigQuery Struct column, the column - should be flattened using "." as the delimiter. Only columns with no child - should have a transformation. - If an input column has no transformations on it, such a column is - ignored by the training, except for the targetColumn, which should have - no transformations defined on. - Only one of column_transformations or column_specs should be passed. - Consider using column_specs as column_transformations will be deprecated eventually. + optimization_objective: Objective function the model is to be optimized towards. The training process creates a Model that optimizes the value of the objective function over the validation set. The supported optimization objectives: "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). "minimize-mae" - Minimize mean-absolute error (MAE). "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). "minimize-rmspe" - Minimize root-mean-squared percentage error (RMSPE). "minimize-wape-mae" - Minimize the combination of weighted absolute percentage error (WAPE) and mean-absolute-error (MAE). "minimize-quantile-loss" - Minimize the quantile loss at the defined quantiles. (Set this objective to build quantile forecasts.) + column_specs: Alternative to column_transformations where the keys of the dict are column names and their respective values are one of AutoMLTabularTrainingJob.column_data_types. When creating transformation for BigQuery Struct column, the column should be flattened using "." as the delimiter. Only columns with no child should have a transformation. If an input column has no transformations on it, such a column is ignored by the training, except for the targetColumn, which should have no transformations defined on. Only one of column_transformations or column_specs should be passed. + column_transformations: Transformations to apply to the input columns (i.e. columns other than the targetColumn). Each transformation may produce multiple result values from the column's value, and all are used for training. When creating transformation for BigQuery Struct column, the column should be flattened using "." as the delimiter. Only columns with no child should have a transformation. If an input column has no transformations on it, such a column is ignored by the training, except for the targetColumn, which should have no transformations defined on. Only one of column_transformations or column_specs should be passed. Consider using column_specs as column_transformations will be deprecated eventually. project: Project to retrieve dataset from. location: Optional location to retrieve dataset from. - labels: The labels with user-defined metadata to - organize TrainingPipelines. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the training pipeline. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this TrainingPipeline will be secured by this key. - Note: Model trained by this TrainingPipeline is also secured - by this key if ``model_to_upload`` is not set separately. - Overrides encryption_spec_key_name set in aiplatform.init. - model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the model. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, the trained Model will be secured by this key. - Overrides encryption_spec_key_name set in aiplatform.init. + labels: The labels with user-defined metadata to organize TrainingPipelines. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the training pipeline. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this TrainingPipeline will be secured by this key. Note: Model trained by this TrainingPipeline is also secured by this key if `model_to_upload` is not set separately. Overrides encryption_spec_key_name set in aiplatform.init. + model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the model. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, the trained Model will be secured by this key. Overrides encryption_spec_key_name set in aiplatform.init. additional_experiments: Additional experiment flags for the time series forcasting training. Returns: - model: The trained Vertex AI Model resource or None if training did not - produce a Vertex AI Model. + model: The trained Vertex AI Model resource or None if training did not produce a Vertex AI Model. """ # fmt: on diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_image_training_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_image_training_job/component.py index 5ade101a9a..7f670e19ef 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_image_training_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_image_training_job/component.py @@ -13,7 +13,7 @@ # limitations under the License. -from typing import Optional +from typing import Dict, List, Optional from google_cloud_pipeline_components import _image from google_cloud_pipeline_components.types.artifact_types import VertexDataset @@ -21,6 +21,7 @@ from kfp import dsl from kfp.dsl import Input from kfp.dsl import Output +from kfp.dsl import OutputPath @dsl.container_component @@ -29,12 +30,18 @@ def automl_image_training_job( display_name: str, dataset: Input[VertexDataset], model: Output[VertexModel], + gcp_resources: OutputPath(str), location: Optional[str] = 'us-central1', prediction_type: Optional[str] = 'classification', multi_label: Optional[bool] = False, model_type: Optional[str] = 'CLOUD', base_model: Optional[Input[VertexModel]] = None, - labels: Optional[dict] = {}, + incremental_train_base_model: Optional[Input[VertexModel]] = None, + parent_model: Optional[Input[VertexModel]] = None, + is_default_version: Optional[bool] = True, + model_version_aliases: Optional[List[str]] = None, + model_version_description: Optional[str] = None, + labels: Optional[Dict[str, str]] = {}, training_encryption_spec_key_name: Optional[str] = None, model_encryption_spec_key_name: Optional[str] = None, training_fraction_split: Optional[float] = None, @@ -45,182 +52,45 @@ def automl_image_training_job( test_filter_split: Optional[str] = None, budget_milli_node_hours: Optional[int] = None, model_display_name: Optional[str] = None, - model_labels: Optional[dict] = None, + model_labels: Optional[Dict[str, str]] = None, disable_early_stopping: Optional[bool] = False, ): # fmt: off """Runs the AutoML Image training job and returns a model. - If training on a Vertex AI dataset, you can use one of the following split configurations: - - Data fraction splits: - Any of ``training_fraction_split``, ``validation_fraction_split`` and - ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If - the provided ones sum to less than 1, the remainder is assigned to sets as - decided by Vertex AI. If none of the fractions are set, by default roughly 80% - of data will be used for training, 10% for validation, and 10% for test. - Data filter splits: - Assigns input data to training, validation, and test sets - based on the given filters, data pieces not matched by any - filter are ignored. Currently only supported for Datasets - containing DataItems. - If any of the filters in this message are to match nothing, then - they can be set as '-' (the minus sign). - If using filter splits, all of ``training_filter_split``, ``validation_filter_split`` and - ``test_filter_split`` must be provided. - - Supported only for unstructured Datasets. + If training on a Vertex AI dataset, you can use one of the following split configurations: Data fraction splits: Any of `training_fraction_split`, `validation_fraction_split` and `test_fraction_split` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Data filter splits: Assigns input data to training, validation, and test sets based on the given filters, data pieces not matched by any filter are ignored. Currently only supported for Datasets containing DataItems. If any of the filters in this message are to match nothing, then they can be set as '-' (the minus sign). If using filter splits, all of `training_filter_split`, `validation_filter_split` and `test_filter_split` must be provided. Supported only for unstructured Datasets. Args: - dataset: The dataset within the same Project from which data will be used to train the Model. The - Dataset must use schema compatible with Model being trained, - and what is compatible should be described in the used - TrainingPipeline's [training_task_definition] - [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. - For tabular Datasets, all their data is exported to - training, to pick and choose from. - training_fraction_split: The fraction of the input data that is to be used to train - the Model. This is ignored if Dataset is not provided. - validation_fraction_split: The fraction of the input data that is to be used to validate - the Model. This is ignored if Dataset is not provided. - test_fraction_split: The fraction of the input data that is to be used to evaluate - the Model. This is ignored if Dataset is not provided. - training_filter_split: A filter on DataItems of the Dataset. DataItems that match - this filter are used to train the Model. A filter with same syntax - as the one used in DatasetService.ListDataItems may be used. If a - single DataItem is matched by more than one of the FilterSplit filters, - then it is assigned to the first set that applies to it in the training, - validation, test order. This is ignored if Dataset is not provided. - Example usage: training_filter_split="labels.aiplatform.googleapis.com/ml_use=training". - validation_filter_split: A filter on DataItems of the Dataset. DataItems that match - this filter are used to validate the Model. A filter with same syntax - as the one used in DatasetService.ListDataItems may be used. If a - single DataItem is matched by more than one of the FilterSplit filters, - then it is assigned to the first set that applies to it in the training, - validation, test order. This is ignored if Dataset is not provided. - Example usage: validation_filter_split= "labels.aiplatform.googleapis.com/ml_use=validation". - test_filter_split: A filter on DataItems of the Dataset. DataItems that match - this filter are used to test the Model. A filter with same syntax - as the one used in DatasetService.ListDataItems may be used. If a - single DataItem is matched by more than one of the FilterSplit filters, - then it is assigned to the first set that applies to it in the training, - validation, test order. This is ignored if Dataset is not provided. - Example usage: test_filter_split= "labels.aiplatform.googleapis.com/ml_use=test". - budget_milli_node_hours: The train budget of creating this Model, expressed in milli node - hours i.e. 1,000 value in this field means 1 node hour. - Defaults by `prediction_type`: - `classification` - For Cloud models the budget must be: 8,000 - 800,000 - milli node hours (inclusive). The default value is 192,000 which - represents one day in wall time, assuming 8 nodes are used. - `object_detection` - For Cloud models the budget must be: 20,000 - 900,000 - milli node hours (inclusive). The default value is 216,000 which represents - one day in wall time, assuming 9 nodes are used. - The training cost of the model will not exceed this budget. The final - cost will be attempted to be close to the budget, though may end up - being (even) noticeably smaller - at the backend's discretion. This - especially may happen when further model training ceases to provide - any improvements. If the budget is set to a value known to be insufficient to - train a Model for the given training set, the training won't be attempted and - will error. - model_display_name: The display name of the managed Vertex AI Model. The name - can be up to 128 characters long and can be consist of any UTF-8 - characters. If not provided upon creation, the job's display_name is used. - model_labels: The labels with user-defined metadata to - organize your Models. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - disable_early_stopping: bool = False - If true, the entire budget is used. This disables the early stopping - feature. By default, the early stopping feature is enabled, which means - that training might stop before the entire training budget has been - used, if further training does no longer brings significant improvement - to the model. + dataset: The dataset within the same Project from which data will be used to train the Model. The Dataset must use schema compatible with Model being trained, and what is compatible should be described in the used TrainingPipeline's [training_task_definition] [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. For tabular Datasets, all their data is exported to training, to pick and choose from. + training_fraction_split: The fraction of the input data that is to be used to train the Model. This is ignored if Dataset is not provided. + validation_fraction_split: The fraction of the input data that is to be used to validate the Model. This is ignored if Dataset is not provided. + test_fraction_split: The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. + training_filter_split: A filter on DataItems of the Dataset. DataItems that match this filter are used to train the Model. A filter with same syntax as the one used in DatasetService.ListDataItems may be used. If a single DataItem is matched by more than one of the FilterSplit filters, then it is assigned to the first set that applies to it in the training, validation, test order. This is ignored if Dataset is not provided. Example usage: training_filter_split="labels.aiplatform.googleapis.com/ml_use=training". + validation_filter_split: A filter on DataItems of the Dataset. DataItems that match this filter are used to validate the Model. A filter with same syntax as the one used in DatasetService.ListDataItems may be used. If a single DataItem is matched by more than one of the FilterSplit filters, then it is assigned to the first set that applies to it in the training, validation, test order. This is ignored if Dataset is not provided. Example usage: validation_filter_split= "labels.aiplatform.googleapis.com/ml_use=validation". + test_filter_split: A filter on DataItems of the Dataset. DataItems that match this filter are used to test the Model. A filter with same syntax as the one used in DatasetService.ListDataItems may be used. If a single DataItem is matched by more than one of the FilterSplit filters, then it is assigned to the first set that applies to it in the training, validation, test order. This is ignored if Dataset is not provided. Example usage: test_filter_split= "labels.aiplatform.googleapis.com/ml_use=test". + budget_milli_node_hours: The train budget of creating this Model, expressed in milli node hours i.e. 1,000 value in this field means 1 node hour. Defaults by `prediction_type`: `classification` - For Cloud models the budget must be: 8,000 - 800,000 milli node hours (inclusive). The default value is 192,000 which represents one day in wall time, assuming 8 nodes are used. `object_detection` - For Cloud models the budget must be: 20,000 - 900,000 milli node hours (inclusive). The default value is 216,000 which represents one day in wall time, assuming 9 nodes are used. The training cost of the model will not exceed this budget. The final cost will be attempted to be close to the budget, though may end up being (even) noticeably smaller - at the backend's discretion. This especially may happen when further model training ceases to provide any improvements. If the budget is set to a value known to be insufficient to train a Model for the given training set, the training won't be attempted and will error. + model_display_name: The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If not provided upon creation, the job's display_name is used. + model_labels: The labels with user-defined metadata to organize your Models. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + disable_early_stopping: If true, the entire budget is used. This disables the early stopping feature. By default, the early stopping feature is enabled, which means that training might stop before the entire training budget has been used, if further training does no longer brings significant improvement to the model. display_name: The user-defined name of this TrainingPipeline. - prediction_type: The type of prediction the Model is to produce, one of: - "classification" - Predict one out of multiple target values is picked for each row. - "object_detection" - Predict a value based on its relation to other values. - This type is available only to columns that contain - semantically numeric values, i.e. integers or floating - point number, even if stored as e.g. strings. - multi_label: bool = False - Default is False. - If false, a single-label (multi-class) Model will be trained - (i.e. assuming that for each image just up to one annotation may be - applicable). If true, a multi-label Model will be trained (i.e. - assuming that for each image multiple annotations may be applicable). - This is only applicable for the "classification" prediction_type and - will be ignored otherwise. - model_type: str = "CLOUD" - One of the following: - "CLOUD" - Default for Image Classification. - A Model best tailored to be used within Google Cloud, and - which cannot be exported. - "CLOUD_HIGH_ACCURACY_1" - Default for Image Object Detection. - A model best tailored to be used within Google Cloud, and - which cannot be exported. Expected to have a higher latency, - but should also have a higher prediction quality than other - cloud models. - "CLOUD_LOW_LATENCY_1" - A model best tailored to be used within - Google Cloud, and which cannot be exported. Expected to have a - low latency, but may have lower prediction quality than other - cloud models. - "MOBILE_TF_LOW_LATENCY_1" - A model that, in addition to being - available within Google Cloud, can also be exported as TensorFlow - or Core ML model and used on a mobile or edge device afterwards. - Expected to have low latency, but may have lower prediction - quality than other mobile models. - "MOBILE_TF_VERSATILE_1" - A model that, in addition to being - available within Google Cloud, can also be exported as TensorFlow - or Core ML model and used on a mobile or edge device with afterwards. - "MOBILE_TF_HIGH_ACCURACY_1" - A model that, in addition to being - available within Google Cloud, can also be exported as TensorFlow - or Core ML model and used on a mobile or edge device afterwards. - Expected to have a higher latency, but should also have a higher - prediction quality than other mobile models. - base_model: Optional[models.Model] = None - Only permitted for Image Classification models. - If it is specified, the new model will be trained based on the `base` model. - Otherwise, the new model will be trained from scratch. The `base` model - must be in the same Project and Location as the new Model to train, - and have the same model_type. + prediction_type: The type of prediction the Model is to produce, one of: "classification" - Predict one out of multiple target values is picked for each row. "object_detection" - Predict a value based on its relation to other values. This type is available only to columns that contain semantically numeric values, i.e. integers or floating point number, even if stored as e.g. strings. + multi_label: Default is False. If false, a single-label (multi-class) Model will be trained (i.e. assuming that for each image just up to one annotation may be applicable). If true, a multi-label Model will be trained (i.e. assuming that for each image multiple annotations may be applicable). This is only applicable for the "classification" prediction_type and will be ignored otherwise. + model_type: One of the following: "CLOUD" - Default for Image Classification. A Model best tailored to be used within Google Cloud, and which cannot be exported. "CLOUD_HIGH_ACCURACY_1" - Default for Image Object Detection. A model best tailored to be used within Google Cloud, and which cannot be exported. Expected to have a higher latency, but should also have a higher prediction quality than other cloud models. "CLOUD_LOW_LATENCY_1" - A model best tailored to be used within Google Cloud, and which cannot be exported. Expected to have a low latency, but may have lower prediction quality than other cloud models. "MOBILE_TF_LOW_LATENCY_1" - A model that, in addition to being available within Google Cloud, can also be exported as TensorFlow or Core ML model and used on a mobile or edge device afterwards. Expected to have low latency, but may have lower prediction quality than other mobile models. "MOBILE_TF_VERSATILE_1" - A model that, in addition to being available within Google Cloud, can also be exported as TensorFlow or Core ML model and used on a mobile or edge device with afterwards. "MOBILE_TF_HIGH_ACCURACY_1" - A model that, in addition to being available within Google Cloud, can also be exported as TensorFlow or Core ML model and used on a mobile or edge device afterwards. Expected to have a higher latency, but should also have a higher prediction quality than other mobile models. + base_model: Only permitted for Image Classification models. If it is specified, the new model will be trained based on the `base` model. Otherwise, the new model will be trained from scratch. The `base` model must be in the same Project and Location as the new Model to train, and have the same model_type. + incremental_train_base_model: Optional for both Image Classification and Object detection models, to incrementally train a new model using an existing model as the starting point, with a reduced training time. If not specified, the new model will be trained from scratch. The `base` model must be in the same Project and Location as the new Model to train, and have the same prediction_type and model_type. + parent_model: The resource name or model ID of an existing model. The new model uploaded by this job will be a version of `parent_model`. Only set this field when training a new version of an existing model. + is_default_version: When set to True, the newly uploaded model version will automatically have alias "default" included. Subsequent uses of the model produced by this job without a version specified will use this "default" version. When set to False, the "default" alias will not be moved. Actions targeting the model version produced by this job will need to specifically reference this version by ID or alias. New model uploads, i.e. version 1, will always be "default" aliased. + model_version_aliases: User provided version aliases so that the model version uploaded by this job can be referenced via alias instead of auto-generated version ID. A default version alias will be created for the first version of the model. The format is [a-z][a-zA-Z0-9-]{0,126}[a-z0-9] + model_version_description: The description of the model version being uploaded by this job. project: Project to retrieve dataset from. location: Optional location to retrieve dataset from. - labels: The labels with user-defined metadata to - organize TrainingPipelines. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the training pipeline. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this TrainingPipeline will be secured by this key. - Note: Model trained by this TrainingPipeline is also secured - by this key if ``model_to_upload`` is not set separately. - Overrides encryption_spec_key_name set in aiplatform.init. - model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the model. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, the trained Model will be secured by this key. - Overrides encryption_spec_key_name set in aiplatform.init. + labels: The labels with user-defined metadata to organize TrainingPipelines. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the training pipeline. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this TrainingPipeline will be secured by this key. Note: Model trained by this TrainingPipeline is also secured by this key if `model_to_upload` is not set separately. Overrides encryption_spec_key_name set in aiplatform.init. + model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the model. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, the trained Model will be secured by this key. Overrides encryption_spec_key_name set in aiplatform.init. Returns: - model: The trained Vertex AI Model resource or None if training did not - produce a Vertex AI Model. + model: The trained Vertex AI Model resource or None if training did not produce a Vertex AI Model. + gcp_resources: Serialized gcp_resources proto tracking the batch prediction job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on @@ -229,100 +99,139 @@ def automl_image_training_job( command=[ 'python3', '-m', - 'google_cloud_pipeline_components.container.v1.aiplatform.remote_runner', - '--cls_name', - 'AutoMLImageTrainingJob', - '--method_name', - 'run', + 'google_cloud_pipeline_components.container.v1.automl_training_job.image.launcher', ], args=[ - '--init.project', + '--type', + 'AutoMLImageTrainingJob', + '--project', project, - '--init.location', + '--location', location, - '--init.display_name', + '--display_name', display_name, - '--init.prediction_type', + '--prediction_type', prediction_type, - '--init.multi_label', + '--multi_label', multi_label, - '--init.model_type', + '--model_type', model_type, - '--init.labels', + '--labels', labels, - '--method.dataset', + '--dataset', dataset.metadata['resourceName'], - '--method.disable_early_stopping', + '--disable_early_stopping', disable_early_stopping, dsl.IfPresentPlaceholder( input_name='training_encryption_spec_key_name', then=[ - '--init.training_encryption_spec_key_name', + '--training_encryption_spec_key_name', training_encryption_spec_key_name, ], ), dsl.IfPresentPlaceholder( input_name='model_encryption_spec_key_name', then=[ - '--init.model_encryption_spec_key_name', + '--model_encryption_spec_key_name', model_encryption_spec_key_name, ], ), dsl.IfPresentPlaceholder( input_name='model_display_name', - then=['--method.model_display_name', model_display_name], + then=['--model_display_name', model_display_name], ), dsl.IfPresentPlaceholder( input_name='training_fraction_split', then=[ - '--method.training_fraction_split', + '--training_fraction_split', training_fraction_split, ], ), dsl.IfPresentPlaceholder( input_name='validation_fraction_split', then=[ - '--method.validation_fraction_split', + '--validation_fraction_split', validation_fraction_split, ], ), dsl.IfPresentPlaceholder( input_name='test_fraction_split', - then=['--method.test_fraction_split', test_fraction_split], + then=['--test_fraction_split', test_fraction_split], ), dsl.IfPresentPlaceholder( input_name='budget_milli_node_hours', then=[ - '--method.budget_milli_node_hours', + '--budget_milli_node_hours', budget_milli_node_hours, ], ), dsl.IfPresentPlaceholder( input_name='training_filter_split', - then=['--method.training_filter_split', training_filter_split], + then=['--training_filter_split', training_filter_split], ), dsl.IfPresentPlaceholder( input_name='validation_filter_split', then=[ - '--method.validation_filter_split', + '--validation_filter_split', validation_filter_split, ], ), dsl.IfPresentPlaceholder( input_name='test_filter_split', - then=['--method.test_filter_split', test_filter_split], + then=['--test_filter_split', test_filter_split], ), dsl.IfPresentPlaceholder( input_name='base_model', then=[ - '--init.base_model', + '--base_model', base_model.metadata['resourceName'], + '--model_labels', + base_model.metadata['labels'], + ], + else_=[ + dsl.IfPresentPlaceholder( + input_name='model_labels', + then=['--model_labels', model_labels], + ) + ], + ), + dsl.IfPresentPlaceholder( + input_name='incremental_train_base_model', + then=[ + '--incremental_train_base_model', + incremental_train_base_model.metadata['resourceName'], ], ), dsl.IfPresentPlaceholder( - input_name='model_labels', - then=['--method.model_labels', model_labels], + input_name='parent_model', + then=[ + '--parent_model', + parent_model.metadata['resourceName'], + ], + ), + dsl.IfPresentPlaceholder( + input_name='is_default_version', + then=[ + '--is_default_version', + is_default_version, + ], + ), + dsl.IfPresentPlaceholder( + input_name='model_version_aliases', + then=[ + '--model_version_aliases', + model_version_aliases, + ], + ), + dsl.IfPresentPlaceholder( + input_name='model_version_description', + then=[ + '--model_version_description', + model_version_description, + ], ), + '--gcp_resources', + gcp_resources, '--executor_input', '{{$}}', '--resource_name_output_artifact_uri', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_tabular_training_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_tabular_training_job/component.py index 1b680aae56..89b057117d 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_tabular_training_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_tabular_training_job/component.py @@ -62,216 +62,44 @@ def automl_tabular_training_job( # fmt: off """Runs the training job and returns a model. - If training on a Vertex AI dataset, you can use one of the following split configurations: - - Data fraction splits: - Any of ``training_fraction_split``, ``validation_fraction_split`` and - ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If - the provided ones sum to less than 1, the remainder is assigned to sets as - decided by Vertex AI. If none of the fractions are set, by default roughly 80% - of data will be used for training, 10% for validation, and 10% for test. - Predefined splits: - Assigns input data to training, validation, and test sets based on the value of a provided key. - If using predefined splits, ``predefined_split_column_name`` must be provided. - Supported only for tabular Datasets. - Timestamp splits: - Assigns input data to training, validation, and test sets - based on a provided timestamps. The youngest data pieces are - assigned to training set, next to validation set, and the oldest - to the test set. - - Supported only for tabular Datasets. + If training on a Vertex AI dataset, you can use one of the following split configurations: Data fraction splits: Any of `training_fraction_split`, `validation_fraction_split` and `test_fraction_split` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Predefined splits: Assigns input data to training, validation, and test sets based on the value of a provided key. If using predefined splits, `predefined_split_column_name` must be provided. Supported only for tabular Datasets. Timestamp splits: Assigns input data to training, validation, and test sets based on a provided timestamps. The youngest data pieces are assigned to training set, next to validation set, and the oldest to the test set. Supported only for tabular Datasets. Args: - dataset: The dataset within the same Project from which data will be used to train the Model. The - Dataset must use schema compatible with Model being trained, - and what is compatible should be described in the used - TrainingPipeline's [training_task_definition] - [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. - For tabular Datasets, all their data is exported to - training, to pick and choose from. + dataset: The dataset within the same Project from which data will be used to train the Model. The Dataset must use schema compatible with Model being trained, and what is compatible should be described in the used TrainingPipeline's [training_task_definition] [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. For tabular Datasets, all their data is exported to training, to pick and choose from. target_column: The name of the column values of which the Model is to predict. - training_fraction_split: The fraction of the input data that is to be used to train - the Model. This is ignored if Dataset is not provided. - validation_fraction_split: The fraction of the input data that is to be used to validate - the Model. This is ignored if Dataset is not provided. - test_fraction_split: The fraction of the input data that is to be used to evaluate - the Model. This is ignored if Dataset is not provided. - predefined_split_column_name: The key is a name of one of the Dataset's data - columns. The value of the key (either the label's value or - value in the column) must be one of {``training``, - ``validation``, ``test``}, and it defines to which set the - given piece of data is assigned. If for a piece of data the - key is not present or has an invalid value, that piece is - ignored by the pipeline. - Supported only for tabular and time series Datasets. - timestamp_split_column_name: The key is a name of one of the Dataset's data - columns. The value of the key values of the key (the values in - the column) must be in RFC 3339 `date-time` format, where - `time-offset` = `"Z"` (e.g. 1985-04-12T23:20:50.52Z). If for a - piece of data the key is not present or has an invalid value, - that piece is ignored by the pipeline. - Supported only for tabular and time series Datasets. - This parameter must be used with training_fraction_split, validation_fraction_split and test_fraction_split. - weight_column: Name of the column that should be used as the weight column. - Higher values in this column give more importance to the row - during Model training. The column must have numeric values between 0 and - 10000 inclusively, and 0 value means that the row is ignored. - If the weight column field is not set, then all rows are assumed to have - equal weight of 1. - budget_milli_node_hours: The train budget of creating this Model, expressed in milli node - hours i.e. 1,000 value in this field means 1 node hour. - The training cost of the model will not exceed this budget. The final - cost will be attempted to be close to the budget, though may end up - being (even) noticeably smaller - at the backend's discretion. This - especially may happen when further model training ceases to provide - any improvements. - If the budget is set to a value known to be insufficient to train a - Model for the given training set, the training won't be attempted and - will error. - The minimum value is 1000 and the maximum is 72000. - model_display_name: If the script produces a managed Vertex AI Model. The display name of - the Model. The name can be up to 128 characters long and can be consist - of any UTF-8 characters. - If not provided upon creation, the job's display_name is used. - model_labels: The labels with user-defined metadata to - organize your Models. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - model_id: The ID to use for the Model produced by this job, - which will become the final component of the model resource name. - This value may be up to 63 characters, and valid characters - are `[a-z0-9_-]`. The first character cannot be a number or hyphen. - parent_model: The resource name or model ID of an existing model. - The new model uploaded by this job will be a version of `parent_model`. - Only set this field when training a new version of an existing model. - is_default_version: When set to True, the newly uploaded model version will - automatically have alias "default" included. Subsequent uses of - the model produced by this job without a version specified will - use this "default" version. - When set to False, the "default" alias will not be moved. - Actions targeting the model version produced by this job will need - to specifically reference this version by ID or alias. - New model uploads, i.e. version 1, will always be "default" aliased. - model_version_aliases: User provided version aliases so that the model version - uploaded by this job can be referenced via alias instead of - auto-generated version ID. A default version alias will be created - for the first version of the model. - The format is [a-z][a-zA-Z0-9-]{0,126}[a-z0-9] + training_fraction_split: The fraction of the input data that is to be used to train the Model. This is ignored if Dataset is not provided. + validation_fraction_split: The fraction of the input data that is to be used to validate the Model. This is ignored if Dataset is not provided. + test_fraction_split: The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. + predefined_split_column_name: The key is a name of one of the Dataset's data columns. The value of the key (either the label's value or value in the column) must be one of {`training`, `validation`, `test`}, and it defines to which set the given piece of data is assigned. If for a piece of data the key is not present or has an invalid value, that piece is ignored by the pipeline. Supported only for tabular and time series Datasets. + timestamp_split_column_name: The key is a name of one of the Dataset's data columns. The value of the key values of the key (the values in the column) must be in RFC 3339 `date-time` format, where `time-offset` = `"Z"` (e.g. 1985-04-12T23:20:50.52Z). If for a piece of data the key is not present or has an invalid value, that piece is ignored by the pipeline. Supported only for tabular and time series Datasets. This parameter must be used with training_fraction_split, validation_fraction_split and test_fraction_split. + weight_column: Name of the column that should be used as the weight column. Higher values in this column give more importance to the row during Model training. The column must have numeric values between 0 and 10000 inclusively, and 0 value means that the row is ignored. If the weight column field is not set, then all rows are assumed to have equal weight of 1. + budget_milli_node_hours: The train budget of creating this Model, expressed in milli node hours i.e. 1,000 value in this field means 1 node hour. The training cost of the model will not exceed this budget. The final cost will be attempted to be close to the budget, though may end up being (even) noticeably smaller - at the backend's discretion. This especially may happen when further model training ceases to provide any improvements. If the budget is set to a value known to be insufficient to train a Model for the given training set, the training won't be attempted and will error. The minimum value is 1000 and the maximum is 72000. + model_display_name: If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If not provided upon creation, the job's display_name is used. + model_labels: The labels with user-defined metadata to organize your Models. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + model_id: The ID to use for the Model produced by this job, which will become the final component of the model resource name. This value may be up to 63 characters, and valid characters are `[a-z0-9_-]`. The first character cannot be a number or hyphen. + parent_model: The resource name or model ID of an existing model. The new model uploaded by this job will be a version of `parent_model`. Only set this field when training a new version of an existing model. + is_default_version: When set to True, the newly uploaded model version will automatically have alias "default" included. Subsequent uses of the model produced by this job without a version specified will use this "default" version. When set to False, the "default" alias will not be moved. Actions targeting the model version produced by this job will need to specifically reference this version by ID or alias. New model uploads, i.e. version 1, will always be "default" aliased. + model_version_aliases: User provided version aliases so that the model version uploaded by this job can be referenced via alias instead of auto-generated version ID. A default version alias will be created for the first version of the model. The format is [a-z][a-zA-Z0-9-]{0,126}[a-z0-9] model_version_description: The description of the model version being uploaded by this job. - disable_early_stopping: If true, the entire budget is used. This disables the early stopping - feature. By default, the early stopping feature is enabled, which means - that training might stop before the entire training budget has been - used, if further training does no longer brings significant improvement - to the model. - export_evaluated_data_items: Whether to export the test set predictions to a BigQuery table. - If False, then the export is not performed. - export_evaluated_data_items_bigquery_destination_uri: URI of desired destination BigQuery table for exported test set predictions. - Expected format: - ``bq://::
`` - If not specified, then results are exported to the following auto-created BigQuery - table: - ``:export_evaluated_examples__.evaluated_examples`` - Applies only if [export_evaluated_data_items] is True. - export_evaluated_data_items_override_destination: Whether to override the contents of [export_evaluated_data_items_bigquery_destination_uri], - if the table exists, for exported test set predictions. If False, and the - table exists, then the training job will fail. - Applies only if [export_evaluated_data_items] is True and - [export_evaluated_data_items_bigquery_destination_uri] is specified. + disable_early_stopping: If true, the entire budget is used. This disables the early stopping feature. By default, the early stopping feature is enabled, which means that training might stop before the entire training budget has been used, if further training does no longer brings significant improvement to the model. + export_evaluated_data_items: Whether to export the test set predictions to a BigQuery table. If False, then the export is not performed. + export_evaluated_data_items_bigquery_destination_uri: URI of desired destination BigQuery table for exported test set predictions. Expected format: `bq://::
` If not specified, then results are exported to the following auto-created BigQuery table: `:export_evaluated_examples__.evaluated_examples` Applies only if [export_evaluated_data_items] is True. + export_evaluated_data_items_override_destination: Whether to override the contents of [export_evaluated_data_items_bigquery_destination_uri], if the table exists, for exported test set predictions. If False, and the table exists, then the training job will fail. Applies only if [export_evaluated_data_items] is True and [export_evaluated_data_items_bigquery_destination_uri] is specified. display_name: The user-defined name of this TrainingPipeline. - optimization_prediction_type: The type of prediction the Model is to produce. - "classification" - Predict one out of multiple target values is - picked for each row. - "regression" - Predict a value based on its relation to other values. - This type is available only to columns that contain - semantically numeric values, i.e. integers or floating - point number, even if stored as e.g. strings. - optimization_objective: Objective function the Model is to be optimized towards. The training - task creates a Model that maximizes/minimizes the value of the objective - function over the validation set. - The supported optimization objectives depend on the prediction type, and - in the case of classification also the number of distinct values in the - target column (two distint values -> binary, 3 or more distinct values - -> multi class). - If the field is not set, the default objective function is used. - Classification: "maximize-au-roc" (default) - Maximize the area under the receiver operating characteristic (ROC) curve. - "minimize-log-loss" - Minimize log loss. - "maximize-au-prc" - Maximize the area under the precision-recall curve. - "maximize-precision-at-recall" - Maximize precision for a specified recall value. - "maximize-recall-at-precision" - Maximize recall for a specified precision value. - Classification (multi class): - "minimize-log-loss" (default) - Minimize log loss. - Regression: - "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). - "minimize-mae" - Minimize mean-absolute error (MAE). - "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). - column_specs: Alternative to column_transformations where the keys of the dict - are column names and their respective values are one of - AutoMLTabularTrainingJob.column_data_types. - When creating transformation for BigQuery Struct column, the column - should be flattened using "." as the delimiter. Only columns with no child - should have a transformation. - If an input column has no transformations on it, such a column is - ignored by the training, except for the targetColumn, which should have - no transformations defined on. - Only one of column_transformations or column_specs should be passed. - column_transformations: - Transformations to apply to the input columns (i.e. columns other - than the targetColumn). Each transformation may produce multiple - result values from the column's value, and all are used for training. - When creating transformation for BigQuery Struct column, the column - should be flattened using "." as the delimiter. Only columns with no child - should have a transformation. - If an input column has no transformations on it, such a column is - ignored by the training, except for the targetColumn, which should have - no transformations defined on. - Only one of column_transformations or column_specs should be passed. - Consider using column_specs as column_transformations will be deprecated eventually. - optimization_objective_recall_value: Required when maximize-precision-at-recall optimizationObjective was - picked, represents the recall value at which the optimization is done. - The minimum value is 0 and the maximum is 1.0. - optimization_objective_precision_value: Required when maximize-recall-at-precision optimizationObjective was - picked, represents the precision value at which the optimization is - done. - The minimum value is 0 and the maximum is 1.0. + optimization_prediction_type: The type of prediction the Model is to produce. "classification" - Predict one out of multiple target values is picked for each row. "regression" - Predict a value based on its relation to other values. This type is available only to columns that contain semantically numeric values, i.e. integers or floating point number, even if stored as e.g. strings. + optimization_objective: Objective function the Model is to be optimized towards. The training task creates a Model that maximizes/minimizes the value of the objective function over the validation set. The supported optimization objectives depend on the prediction type, and in the case of classification also the number of distinct values in the target column (two distint values -> binary, 3 or more distinct values -> multi class). If the field is not set, the default objective function is used. Classification: "maximize-au-roc" (default) - Maximize the area under the receiver operating characteristic (ROC) curve. "minimize-log-loss" - Minimize log loss. "maximize-au-prc" - Maximize the area under the precision-recall curve. "maximize-precision-at-recall" - Maximize precision for a specified recall value. "maximize-recall-at-precision" - Maximize recall for a specified precision value. Classification (multi class): "minimize-log-loss" (default) - Minimize log loss. Regression: "minimize-rmse" (default) - Minimize root-mean-squared error (RMSE). "minimize-mae" - Minimize mean-absolute error (MAE). "minimize-rmsle" - Minimize root-mean-squared log error (RMSLE). + column_specs: Alternative to column_transformations where the keys of the dict are column names and their respective values are one of AutoMLTabularTrainingJob.column_data_types. When creating transformation for BigQuery Struct column, the column should be flattened using "." as the delimiter. Only columns with no child should have a transformation. If an input column has no transformations on it, such a column is ignored by the training, except for the targetColumn, which should have no transformations defined on. Only one of column_transformations or column_specs should be passed. + column_transformations: Transformations to apply to the input columns (i.e. columns other than the targetColumn). Each transformation may produce multiple result values from the column's value, and all are used for training. When creating transformation for BigQuery Struct column, the column should be flattened using "." as the delimiter. Only columns with no child should have a transformation. If an input column has no transformations on it, such a column is ignored by the training, except for the targetColumn, which should have no transformations defined on. Only one of column_transformations or column_specs should be passed. Consider using column_specs as column_transformations will be deprecated eventually. + optimization_objective_recall_value: Required when maximize-precision-at-recall optimizationObjective was picked, represents the recall value at which the optimization is done. The minimum value is 0 and the maximum is 1.0. + optimization_objective_precision_value: Required when maximize-recall-at-precision optimizationObjective was picked, represents the precision value at which the optimization is done. The minimum value is 0 and the maximum is 1.0. project: Project to retrieve dataset from. location: Optional location to retrieve dataset from. - labels: The labels with user-defined metadata to - organize TrainingPipelines. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the training pipeline. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this TrainingPipeline will be secured by this key. - Note: Model trained by this TrainingPipeline is also secured - by this key if ``model_to_upload`` is not set separately. - Overrides encryption_spec_key_name set in aiplatform.init. - model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the model. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, the trained Model will be secured by this key. - Overrides encryption_spec_key_name set in aiplatform.init. + labels: The labels with user-defined metadata to organize TrainingPipelines. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the training pipeline. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this TrainingPipeline will be secured by this key. Note: Model trained by this TrainingPipeline is also secured by this key if `model_to_upload` is not set separately. Overrides encryption_spec_key_name set in aiplatform.init. + model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the model. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, the trained Model will be secured by this key. Overrides encryption_spec_key_name set in aiplatform.init. Returns: - model: The trained Vertex AI Model resource or None if training did not - produce a Vertex AI Model. + model: The trained Vertex AI Model resource or None if training did not produce a Vertex AI Model. """ # fmt: on diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_text_training_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_text_training_job/component.py index ba21409c61..6ad67e24d0 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_text_training_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_text_training_job/component.py @@ -45,99 +45,24 @@ def automl_text_training_job( # fmt: off """Runs the training job and returns a model. - If training on a Vertex AI dataset, you can use one of the following split configurations: - - Data fraction splits: - Any of ``training_fraction_split``, ``validation_fraction_split`` and - ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If - the provided ones sum to less than 1, the remainder is assigned to sets as - decided by Vertex AI. If none of the fractions are set, by default roughly 80% - of data will be used for training, 10% for validation, and 10% for test. - Data filter splits: - Assigns input data to training, validation, and test sets - based on the given filters, data pieces not matched by any - filter are ignored. Currently only supported for Datasets - containing DataItems. - If any of the filters in this message are to match nothing, then - they can be set as '-' (the minus sign). - - Supported only for unstructured Datasets. + If training on a Vertex AI dataset, you can use one of the following split configurations: Data fraction splits: Any of `training_fraction_split`, `validation_fraction_split` and `test_fraction_split` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Data filter splits: Assigns input data to training, validation, and test sets based on the given filters, data pieces not matched by any filter are ignored. Currently only supported for Datasets containing DataItems. If any of the filters in this message are to match nothing, then they can be set as '-' (the minus sign). Supported only for unstructured Datasets. Args: - dataset: The dataset within the same Project from which data will be used to train the Model. The - Dataset must use schema compatible with Model being trained, - and what is compatible should be described in the used - TrainingPipeline's [training_task_definition] - [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. - training_fraction_split: The fraction of the input data that is to be used to train - the Model. This is ignored if Dataset is not provided. - validation_fraction_split: The fraction of the input data that is to be used to validate - the Model. This is ignored if Dataset is not provided. - test_fraction_split: The fraction of the input data that is to be used to evaluate - the Model. This is ignored if Dataset is not provided. - model_display_name: The display name of the managed Vertex AI Model. - The name can be up to 128 characters long and can consist - of any UTF-8 characters. - If not provided upon creation, the job's display_name is used. - model_labels: The labels with user-defined metadata to - organize your Models. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. + dataset: The dataset within the same Project from which data will be used to train the Model. The Dataset must use schema compatible with Model being trained, and what is compatible should be described in the used TrainingPipeline's [training_task_definition] [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. + training_fraction_split: The fraction of the input data that is to be used to train the Model. This is ignored if Dataset is not provided. + validation_fraction_split: The fraction of the input data that is to be used to validate the Model. This is ignored if Dataset is not provided. + test_fraction_split: The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. + model_display_name: The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can consist of any UTF-8 characters. If not provided upon creation, the job's display_name is used. + model_labels: The labels with user-defined metadata to organize your Models. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. display_name: The user-defined name of this TrainingPipeline. - prediction_type: The type of prediction the Model is to produce, one of: - "classification" - A classification model analyzes text data and returns a list of categories that apply to the text found in the data. - Vertex AI offers both single-label and multi-label text classification models. - "extraction" - An entity extraction model inspects text data known entities referenced in the data and labels those entities in the text. - "sentiment" - A sentiment analysis model inspects text data and identifies the prevailing emotional opinion within it, especially to determine a writer's attitude as positive, negative, or neutral. - multi_label: Required and only applicable for text classification task. If false, a single-label (multi-class) Model will be trained (i.e. - assuming that for each text snippet just up to one annotation may be - applicable). If true, a multi-label Model will be trained (i.e. - assuming that for each text snippet multiple annotations may be - applicable). - sentiment_max: Required and only applicable for sentiment task. A sentiment is expressed as an integer - ordinal, where higher value means a more - positive sentiment. The range of sentiments that - will be used is between 0 and sentimentMax - (inclusive on both ends), and all the values in - the range must be represented in the dataset - before a model can be created. - Only the Annotations with this sentimentMax will - be used for training. sentimentMax value must be - between 1 and 10 (inclusive). + prediction_type: The type of prediction the Model is to produce, one of: "classification" - A classification model analyzes text data and returns a list of categories that apply to the text found in the data. Vertex AI offers both single-label and multi-label text classification models. "extraction" - An entity extraction model inspects text data known entities referenced in the data and labels those entities in the text. "sentiment" - A sentiment analysis model inspects text data and identifies the prevailing emotional opinion within it, especially to determine a writer's attitude as positive, negative, or neutral. + multi_label: Required and only applicable for text classification task. If false, a single-label (multi-class) Model will be trained (i.e. assuming that for each text snippet just up to one annotation may be applicable). If true, a multi-label Model will be trained (i.e. assuming that for each text snippet multiple annotations may be applicable). + sentiment_max: Required and only applicable for sentiment task. A sentiment is expressed as an integer ordinal, where higher value means a more positive sentiment. The range of sentiments that will be used is between 0 and sentimentMax (inclusive on both ends), and all the values in the range must be represented in the dataset before a model can be created. Only the Annotations with this sentimentMax will be used for training. sentimentMax value must be between 1 and 10 (inclusive). project: Project to retrieve dataset from. location: Optional location to retrieve dataset from. - labels: The labels with user-defined metadata to - organize TrainingPipelines. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the training pipeline. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this TrainingPipeline will be secured by this key. - Note: Model trained by this TrainingPipeline is also secured - by this key if ``model_to_upload`` is not set separately. - Overrides encryption_spec_key_name set in aiplatform.init. - model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the model. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, the trained Model will be secured by this key. - Overrides encryption_spec_key_name set in aiplatform.init. + labels: The labels with user-defined metadata to organize TrainingPipelines. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the training pipeline. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this TrainingPipeline will be secured by this key. Note: Model trained by this TrainingPipeline is also secured by this key if `model_to_upload` is not set separately. Overrides encryption_spec_key_name set in aiplatform.init. + model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the model. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, the trained Model will be secured by this key. Overrides encryption_spec_key_name set in aiplatform.init. Returns: model: The trained Vertex AI Model resource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_video_training_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_video_training_job/component.py index 62c8484f20..6818d02577 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_video_training_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/automl/training_job/automl_video_training_job/component.py @@ -43,108 +43,25 @@ def automl_video_training_job( # fmt: off """Runs the AutoML Video training job and returns a model. - If training on a Vertex AI dataset, you can use one of the following split configurations: - - Data fraction splits: - ``training_fraction_split``, and ``test_fraction_split`` may optionally - be provided, they must sum to up to 1. If none of the fractions are set, - by default roughly 80% of data will be used for training, and 20% for test. - Data filter splits: - Assigns input data to training, validation, and test sets - based on the given filters, data pieces not matched by any - filter are ignored. Currently only supported for Datasets - containing DataItems. - If any of the filters in this message are to match nothing, then - they can be set as '-' (the minus sign). - - Supported only for unstructured Datasets. + If training on a Vertex AI dataset, you can use one of the following split configurations: Data fraction splits: `training_fraction_split`, and `test_fraction_split` may optionally be provided, they must sum to up to 1. If none of the fractions are set, by default roughly 80% of data will be used for training, and 20% for test. Data filter splits: Assigns input data to training, validation, and test sets based on the given filters, data pieces not matched by any filter are ignored. Currently only supported for Datasets containing DataItems. If any of the filters in this message are to match nothing, then they can be set as '-' (the minus sign). Supported only for unstructured Datasets. Args: - dataset: The dataset within the same Project from which data will be used to train the Model. The - Dataset must use schema compatible with Model being trained, - and what is compatible should be described in the used - TrainingPipeline's [training_task_definition] - [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. - For tabular Datasets, all their data is exported to - training, to pick and choose from. - training_fraction_split: The fraction of the input data that is to be used to train - the Model. This is ignored if Dataset is not provided. - test_fraction_split: The fraction of the input data that is to be used to evaluate - the Model. This is ignored if Dataset is not provided. - model_display_name: The display name of the managed Vertex AI Model. The name - can be up to 128 characters long and can be consist of any UTF-8 - characters. If not provided upon creation, the job's display_name is used. - model_labels: The labels with user-defined metadata to - organize your Models. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. + dataset: The dataset within the same Project from which data will be used to train the Model. The Dataset must use schema compatible with Model being trained, and what is compatible should be described in the used TrainingPipeline's [training_task_definition] [google.cloud.aiplatform.v1beta1.TrainingPipeline.training_task_definition]. For tabular Datasets, all their data is exported to training, to pick and choose from. + training_fraction_split: The fraction of the input data that is to be used to train the Model. This is ignored if Dataset is not provided. + test_fraction_split: The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. + model_display_name: The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If not provided upon creation, the job's display_name is used. + model_labels: The labels with user-defined metadata to organize your Models. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. display_name: The user-defined name of this TrainingPipeline. - prediction_type: The type of prediction the Model is to produce, one of: - "classification" - A video classification model classifies shots and segments in your videos according to your own defined labels. - "object_tracking" - A video object tracking model detects and tracks multiple objects in shots and segments. You can use these models to track objects in your videos according to your own pre-defined, custom labels. - "action_recognition" - A video action reconition model pinpoints the location of actions with short temporal durations (~1 second). - model_type: str = "CLOUD" - One of the following: - "CLOUD" - available for "classification", "object_tracking" and "action_recognition" - A Model best tailored to be used within Google Cloud, - and which cannot be exported. - "MOBILE_VERSATILE_1" - available for "classification", "object_tracking" and "action_recognition" - A model that, in addition to being available within Google - Cloud, can also be exported (see ModelService.ExportModel) - as a TensorFlow or TensorFlow Lite model and used on a - mobile or edge device with afterwards. - "MOBILE_CORAL_VERSATILE_1" - available only for "object_tracking" - A versatile model that is meant to be exported (see - ModelService.ExportModel) and used on a Google Coral device. - "MOBILE_CORAL_LOW_LATENCY_1" - available only for "object_tracking" - A model that trades off quality for low latency, to be - exported (see ModelService.ExportModel) and used on a - Google Coral device. - "MOBILE_JETSON_VERSATILE_1" - available only for "object_tracking" - A versatile model that is meant to be exported (see - ModelService.ExportModel) and used on an NVIDIA Jetson device. - "MOBILE_JETSON_LOW_LATENCY_1" - available only for "object_tracking" - A model that trades off quality for low latency, to be - exported (see ModelService.ExportModel) and used on an - NVIDIA Jetson device. + prediction_type: The type of prediction the Model is to produce, one of: "classification" - A video classification model classifies shots and segments in your videos according to your own defined labels. "object_tracking" - A video object tracking model detects and tracks multiple objects in shots and segments. You can use these models to track objects in your videos according to your own pre-defined, custom labels. "action_recognition" - A video action reconition model pinpoints the location of actions with short temporal durations (~1 second). + model_type: str = "CLOUD" One of the following: "CLOUD" - available for "classification", "object_tracking" and "action_recognition" A Model best tailored to be used within Google Cloud, and which cannot be exported. "MOBILE_VERSATILE_1" - available for "classification", "object_tracking" and "action_recognition" A model that, in addition to being available within Google Cloud, can also be exported (see ModelService.ExportModel) as a TensorFlow or TensorFlow Lite model and used on a mobile or edge device with afterwards. "MOBILE_CORAL_VERSATILE_1" - available only for "object_tracking" A versatile model that is meant to be exported (see ModelService.ExportModel) and used on a Google Coral device. "MOBILE_CORAL_LOW_LATENCY_1" - available only for "object_tracking" A model that trades off quality for low latency, to be exported (see ModelService.ExportModel) and used on a Google Coral device. "MOBILE_JETSON_VERSATILE_1" - available only for "object_tracking" A versatile model that is meant to be exported (see ModelService.ExportModel) and used on an NVIDIA Jetson device. "MOBILE_JETSON_LOW_LATENCY_1" - available only for "object_tracking" A model that trades off quality for low latency, to be exported (see ModelService.ExportModel) and used on an NVIDIA Jetson device. project: Project to retrieve dataset from. location: Optional location to retrieve dataset from. - labels: The labels with user-defined metadata to - organize TrainingPipelines. - Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only - contain lowercase letters, numeric characters, - underscores and dashes. International characters - are allowed. - See https://goo.gl/xmQnxf for more information - and examples of labels. - training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the training pipeline. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this TrainingPipeline will be secured by this key. - Note: Model trained by this TrainingPipeline is also secured - by this key if ``model_to_upload`` is not set separately. - Overrides encryption_spec_key_name set in aiplatform.init. - model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the model. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, the trained Model will be secured by this key. - Overrides encryption_spec_key_name set in aiplatform.init. + labels: The labels with user-defined metadata to organize TrainingPipelines. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + training_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the training pipeline. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this TrainingPipeline will be secured by this key. Note: Model trained by this TrainingPipeline is also secured by this key if `model_to_upload` is not set separately. Overrides encryption_spec_key_name set in aiplatform.init. + model_encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the model. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, the trained Model will be secured by this key. Overrides encryption_spec_key_name set in aiplatform.init. Returns: - model: The trained Vertex AI Model resource or None if training did not - produce a Vertex AI Model. + model: The trained Vertex AI Model resource or None if training did not produce a Vertex AI Model. """ # fmt`:` on diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/__init__.py index 9cc8710051..bc8b2730f2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Serve batch predictions from your models using `Vertex AI Batch Predictions `_.""" +# fmt: off +"""Serve batch predictions from your models using [Vertex AI Batch Predictions](https://cloud.google.com/vertex-ai/docs/predictions/overview?_ga=2.161419069.-1686833729.1684288907#batch_predictions).""" +# fmt: on from google_cloud_pipeline_components.v1.batch_predict_job.component import model_batch_predict as ModelBatchPredictOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/component.py index 5d2fdb2c0b..235632ac21 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/batch_predict_job/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQTable from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel from google_cloud_pipeline_components.types.artifact_types import VertexBatchPredictionJob @@ -31,7 +32,6 @@ @container_component def model_batch_predict( - project: str, job_display_name: str, gcp_resources: OutputPath(str), batchpredictionjob: Output[VertexBatchPredictionJob], @@ -56,241 +56,53 @@ def model_batch_predict( accelerator_count: int = 0, starting_replica_count: int = 0, max_replica_count: int = 0, + service_account: str = '', manual_batch_tuning_parameters_batch_size: int = 0, generate_explanation: bool = False, explanation_metadata: Dict[str, str] = {}, explanation_parameters: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Creates a Google Cloud Vertex BatchPredictionJob and waits for it to - complete. - - For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs/create. + """Creates a Google Cloud Vertex [BatchPredictionJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs) and waits for it to complete. For more details, see [BatchPredictionJob.Create](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs/create). Args: - project: Project to create the BatchPredictionJob. job_display_name: The user-defined name of this BatchPredictionJob. location: Location for creating the BatchPredictionJob. - If not set, default to us-central1. - instances_format: The format in which instances are - given, must be one of the Model's supportedInputStorageFormats. If not - set, default to "jsonl". For more details about this input config, - see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - predictions_format: The format in which Vertex AI gives the predictions. Must be one of the - Model's supportedOutputStorageFormats. If not set, default to "jsonl". - For more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - model: The Model used to get predictions via this job. Must share the same - ancestor Location. Starting this job has no impact on any existing - deployments of the Model and their resources. Either this or - unmanaged_container_model must be specified. - unmanaged_container_model: The unmanaged container model used to get predictions via this job. - This should be used for models that are not uploaded to Vertex. Either - this or model must be specified. - gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction - on. They must match `instances_format`. May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - bigquery_source_input_uri: BigQuery URI to a table, up to 2000 characters long. For example: - `projectId.bqDatasetId.bqTableId` For more details about this input - config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + instances_format: The format in which instances are given, must be one of the [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models)'s supportedInputStorageFormats. For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig.) + predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see [OutputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig). + model: The Model used to get predictions via this job. Must share the same ancestor Location. Starting this job has no impact on any existing deployments of the Model and their resources. Either this or `unmanaged_container_model` must be specified. + unmanaged_container_model: The unmanaged container model used to get predictions via this job. This should be used for models that are not uploaded to Vertex. Either this or model must be specified. + gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. They must match `instances_format`. May contain wildcards. For more information on wildcards, see [WildcardNames](https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames). For more details about this input config, see [InputConfig](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig). + bigquery_source_input_uri: BigQuery URI to a table, up to 2000 characters long. For example: `projectId.bqDatasetId.bqTableId` For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. model_parameters: The parameters that govern the predictions. The schema of the parameters - instance_type (Optional[str]): - The format of the instance that the Model accepts. Vertex AI will - convert compatible - [batch prediction input instance formats][InputConfig.instances_format] - to the specified format. - Supported values are: - ** `object`: Each input is converted to JSON object format. - * For `bigquery`, each row is converted to an object. - * For `jsonl`, each line of the JSONL input must be an object. - * Does not apply to `csv`, `file-list`, `tf-record`, or - `tf-record-gzip`. - ** `array`: Each input is converted to JSON array format. - * For `bigquery`, each row is converted to an array. The order - of columns is determined by the BigQuery column order, unless - [included_fields][] is populated. - [included_fields][] must be populated for specifying field orders. - * For `jsonl`, if each line of the JSONL input is an object, - [included_fields][] must be populated for specifying field orders. - * Does not apply to `csv`, `file-list`, `tf-record`, or - `tf-record-gzip`. - If not specified, Vertex AI converts the batch prediction input as - follows: - * For `bigquery` and `csv`, the behavior is the same as `array`. The - order of columns is the same as defined in the file or table, unless - [included_fields][] is populated. - * For `jsonl`, the prediction instance format is determined by - each line of the input. - * For `tf-record`/`tf-record-gzip`, each record will be converted to - an object in the format of `{"b64": }`, where `` is - the Base64-encoded string of the content of the record. - * For `file-list`, each file in the list will be converted to an - object in the format of `{"b64": }`, where `` is - the Base64-encoded string of the content of the file. - (-- api-linter: core::0140::base64=disabled - aip.dev/not-precedent: Base64 is not for this field. --) - key_field (Optional[str]): - The name of the field that is considered as a key. - The values identified by the key field is not included in the - transformed instances that is sent to the Model. This is similar to - specifying this name of the field in [excluded_fields][]. In addition, - the batch prediction output will not include the instances. Instead the - output will only include the value of the key field, in a field named - `key` in the output: - * For `jsonl` output format, the output will have a `key` field - instead of the `instance` field. - * For `csv`/`bigquery` output format, the output will have have a `key` - column instead of the instance feature columns. - The input must be JSONL with objects at each line, CSV, BigQuery - or TfRecord. - included_fields (Optional[Sequence[str]]): - Fields that will be included in the prediction instance that is - sent to the Model. - If [instance_type][] is `array`, the order of field names in - included_fields also determines the order of the values in the array. - When included_fields is populated, [excluded_fields][] must be empty. - The input must be JSONL with objects at each line, CSV, BigQuery - or TfRecord. - excluded_fields (Optional[Sequence[str]]): - Fields that will be excluded in the prediction instance that is - sent to the Model. - Excluded will be attached to the batch prediction output if - [key_field][] is not specified. - When excluded_fields is populated, [included_fields][] must be empty. - The input must be JSONL with objects at each line, CSV, BigQuery - or TfRecord. - may be specified via the Model's `parameters_schema_uri`. - gcs_destination_output_uri_prefix: The Google Cloud - Storage location of the directory where the output is to be written - to. In the given directory a new directory is created. Its name is - ``prediction--``, where timestamp - is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files - ``predictions_0001.``, ``predictions_0002.``, - ..., ``predictions_N.`` are created where ```` - depends on chosen ``predictions_format``, and N may equal 0001 and - depends on the total number of successfully predicted instances. If - the Model has both ``instance`` and ``prediction`` schemata defined - then each such file contains predictions as per the - ``predictions_format``. If prediction for any instance failed - (partially or completely), then an additional - ``errors_0001.``, ``errors_0002.``,..., - ``errors_N.`` files are created (N depends on total number - of failed predictions). These files contain the failed instances, as - per their schema, followed by an additional ``error`` field which as - value has ``google.rpc.Status`` containing only ``code`` and - ``message`` fields. For more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In - the given project a new dataset is created with name - ``prediction__`` where is made - BigQuery-dataset-name compatible (for example, most special characters - become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ - "based on ISO-8601" format. In the dataset two tables will be created, - ``predictions``, and ``errors``. If the Model has both ``instance`` - and ``prediction`` schemata defined then the tables have columns as - follows: The ``predictions`` table contains instances for which the - prediction succeeded, it has columns as per a concatenation of the - Model's instance and prediction schemata. The ``errors`` table - contains rows for which the prediction has failed, it has instance - columns, as per the instance schema, followed by a single "errors" - column, which as values has ```google.rpc.Status`` `__ - represented as a STRUCT, and containing only ``code`` and - ``message``. For more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - machine_type: The type of machine for running batch - prediction on dedicated resources. If the Model supports - DEDICATED_RESOURCES this config may be provided (and the job will use - these resources). If the Model doesn't support AUTOMATIC_RESOURCES, - this config must be provided. For more details about the - BatchDedicatedResources, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. - For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - accelerator_type: The type of accelerator(s) that may be - attached to the machine as per `accelerator_count`. Only used if - `machine_type` is set. For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - accelerator_count: The number of accelerators to attach - to the `machine_type`. Only used if `machine_type` is set. For more - details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - starting_replica_count: The number of machine replicas - used at the start of the batch operation. If not set, Vertex AI - decides starting number, not greater than `max_replica_count`. Only - used if `machine_type` is set. - max_replica_count: The maximum number of machine replicas the batch operation may be scaled - to. Only used if `machine_type` is set. Default is 10. - manual_batch_tuning_parameters_batch_size: The number of - the records (e.g. instances) of the operation given in each batch to a - machine replica. Machine type, and size of a single record should be - considered when setting this parameter, higher value speeds up the - batch operation's execution, but too high value will result in a whole - batch not fitting in a machine's memory, and the whole operation will - fail. The default value is 4. - generate_explanation: Generate explanation along with - the batch prediction results. This will cause the batch prediction - output to include explanations based on the `prediction_format`: - - `bigquery`: output includes a column named `explanation`. The value is - a struct that conforms to the [aiplatform.gapic.Explanation] object. - - `jsonl`: The JSON objects on each line include an additional entry - keyed `explanation`. The value of the entry is a JSON object that - conforms to the [aiplatform.gapic.Explanation] object. - `csv`: - Generating explanations for CSV format is not supported. If this - field is set to true, either the Model.explanation_spec or - explanation_metadata and explanation_parameters must be populated. - explanation_metadata: Explanation metadata - configuration for this BatchPredictionJob. Can be specified only if - `generate_explanation` is set to `True`. This value overrides the - value of `Model.explanation_metadata`. All fields of - `explanation_metadata` are optional in the request. If a field of the - `explanation_metadata` object is not populated, the corresponding - field of the `Model.explanation_metadata` object is inherited. For - more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. - explanation_parameters: Parameters to configure - explaining for Model's predictions. Can be specified only if - `generate_explanation` is set to `True`. This value overrides the - value of `Model.explanation_parameters`. All fields of - `explanation_parameters` are optional in the request. If a field of - the `explanation_parameters` object is not populated, the - corresponding field of the `Model.explanation_parameters` object is - inherited. For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. - labels: The labels with user-defined metadata to - organize your BatchPredictionJobs. Label keys and values can be no - longer than 64 characters (Unicode codepoints), can only contain - lowercase letters, numeric characters, underscores and dashes. - International characters are allowed. See https://goo.gl/xmQnxf for - more information and examples of labels. - encryption_spec_key_name: Customer-managed encryption - key options for a BatchPredictionJob. If this is set, then all - resources created by the BatchPredictionJob will be encrypted with the - provided encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource - is created. + instance_type: The format of the instance that the Model accepts. Vertex AI will convert compatible [InstancesFormat](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig) to the specified format. Supported values are: `object`: Each input is converted to JSON object format. * For `bigquery`, each row is converted to an object. * For `jsonl`, each line of the JSONL input must be an object. * Does not apply to `csv`, `file-list`, `tf-record`, or `tf-record-gzip`. `array`: Each input is converted to JSON array format. * For `bigquery`, each row is converted to an array. The order of columns is determined by the BigQuery column order, unless [included_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig) is populated. `included_fields` must be populated for specifying field orders. * For `jsonl`, if each line of the JSONL input is an object, `included_fields` must be populated for specifying field orders. * Does not apply to `csv`, `file-list`, `tf-record`, or `tf-record-gzip`. If not specified, Vertex AI converts the batch prediction input as follows: * For `bigquery` and `csv`, the behavior is the same as `array`. The order of columns is the same as defined in the file or table, unless included_fields is populated. * For `jsonl`, the prediction instance format is determined by each line of the input. * For `tf-record`/`tf-record-gzip`, each record will be converted to an object in the format of `{"b64": }`, where `` is the Base64-encoded string of the content of the record. * For `file-list`, each file in the list will be converted to an object in the format of `{"b64": }`, where `` is the Base64-encoded string of the content of the file. + key_field: The name of the field that is considered as a key. The values identified by the key field is not included in the transformed instances that is sent to the Model. This is similar to specifying this name of the field in [excluded_fields](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig). In addition, the batch prediction output will not include the instances. Instead the output will only include the value of the key field, in a field named `key` in the output: * For `jsonl` output format, the output will have a `key` field instead of the `instance` field. * For `csv`/`bigquery` output format, the output will have have a `key` column instead of the instance feature columns. The input must be JSONL with objects at each line, CSV, BigQuery or TfRecord. + included_fields: Fields that will be included in the prediction instance that is sent to the Model. If `instance_type` is `array`, the order of field names in `included_fields` also determines the order of the values in the array. When `included_fields` is populated, `excluded_fields` must be empty. The input must be JSONL with objects at each line, CSV, BigQuery or TfRecord. + excluded_fields: Fields that will be excluded in the prediction instance that is sent to the Model. Excluded will be attached to the batch prediction output if key_field is not specified. When `excluded_fields` is populated, `included_fields` must be empty. The input must be JSONL with objects at each line, CSV, BigQuery or TfRecord. may be specified via the Model's `parameters_schema_uri`. + gcs_destination_output_uri_prefix: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has [google.rpc.Status](Status) represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + accelerator_type: The type of accelerator(s) that may be attached to the machine as per `accelerator_count`. Only used if `machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + accelerator_count: The number of accelerators to attach to the `machine_type`. Only used if `machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + service_account: The service account that the DeployedModel's container runs as. If not specified, a system generated one will be used, which has minimal permissions and the custom container, if used, may not have enough permission to access other Google Cloud resources. Users deploying the Model must have the iam.serviceAccounts.actAs permission on this service account. + manual_batch_tuning_parameters_batch_size: The number of the records (e.g. instances) of the operation given in each batch to a machine replica. Machine type, and size of a single record should be considered when setting this parameter, higher value speeds up the batch operation's execution, but too high value will result in a whole batch not fitting in a machine's memory, and the whole operation will fail. + generate_explanation: Generate explanation along with the batch prediction results. This will cause the batch prediction output to include explanations based on the `prediction_format`: - `bigquery`: output includes a column named `explanation`. The value is a struct that conforms to the [aiplatform.gapic.Explanation] object. - `jsonl`: The JSON objects on each line include an additional entry keyed `explanation`. The value of the entry is a JSON object that conforms to the [aiplatform.gapic.Explanation] object. - `csv`: Generating explanations for CSV format is not supported. If this field is set to true, either the Model.explanation_spec or explanation_metadata and explanation_parameters must be populated. + explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + labels: The labels with user-defined metadata to organize your BatchPredictionJobs. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + encryption_spec_key_name: Customer-managed encryption key options for a BatchPredictionJob. If this is set, then all resources created by the BatchPredictionJob will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + project: Project to create the BatchPredictionJob. Defaults to the project in which the PipelineJob is run. Returns: - batchpredictionjob: [**Deprecated. Use gcs_output_directory and bigquery_output_table - instead.**] Artifact - representation of the created batch prediction job. - gcs_output_directory: Artifact tracking the batch prediction job output. This is only - available if - gcs_destination_output_uri_prefix is specified. - bigquery_output_table: Artifact tracking the batch prediction job output. This is only - available if - bigquery_output_table is specified. - gcp_resources: Serialized gcp_resources proto tracking the batch prediction job. - - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + batchpredictionjob: [**Deprecated. Use gcs_output_directory and bigquery_output_table instead.**] Artifact representation of the created batch prediction job. + gcs_output_directory: Artifact tracking the batch prediction job output. This is only available if gcs_destination_output_uri_prefix is specified. + bigquery_output_table: Artifact tracking the batch prediction job output. This is only available if bigquery_output_table is specified. + gcp_resources: Serialized gcp_resources proto tracking the batch prediction job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( @@ -387,6 +199,9 @@ def model_batch_predict( ', "max_replica_count": ', max_replica_count, '}', + ', "service_account": "', + service_account, + '"', ', "manual_batch_tuning_parameters": {', '"batch_size": ', manual_batch_tuning_parameters_batch_size, diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/__init__.py index a40784d810..de0fb5cf3f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Create and execute machine learning models via SQL using `Google Cloud BigQuery ML `_.""" +# fmt: off +"""Create and execute machine learning models via SQL using [Google Cloud BigQuery ML](https://cloud.google.com/bigquery/docs/bqml-introduction).""" +# fmt: on from google_cloud_pipeline_components.v1.bigquery.create_model.component import bigquery_create_model_job as BigqueryCreateModelJobOp from google_cloud_pipeline_components.v1.bigquery.detect_anomalies_model.component import bigquery_detect_anomalies_job as BigqueryDetectAnomaliesModelJobOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/create_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/create_model/component.py index 77852d3935..7b36700e47 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/create_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/create_model/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component @@ -25,7 +26,6 @@ @container_component def bigquery_create_model_job( - project: str, query: str, model: Output[BQMLModel], gcp_resources: OutputPath(str), @@ -33,37 +33,23 @@ def bigquery_create_model_job( query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery create model job and waits for it to finish. Args: - project: Project to run BigQuery model creation job. - location: Location of the job to create the BigQuery model. If not set, default to - `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - query: SQL query text to execute. Only standard SQL is - supported. If query are both specified in here and in - job_configuration_query, the value in here will override the other - one. - query_parameters: Query parameters for standard SQL queries. - If query_parameters are both specified in here and in - job_configuration_query, the value in here will override the other one. - job_configuration_query: A json formatted string describing the rest of the job configuration. - For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location of the job to create the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + query: SQL query text to execute. Only standard SQL is supported. If query are both specified in here and in job_configuration_query, the value in here will override the other one. + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery model creation job. Defaults to the project in which the PipelineJob is run. Returns: model: Describes the model which is created. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/detect_anomalies_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/detect_anomalies_model/component.py index 022b2333c7..d281320922 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/detect_anomalies_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/detect_anomalies_model/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_detect_anomalies_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -40,68 +40,28 @@ def bigquery_detect_anomalies_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery detect anomalies model job and waits for it to finish. Args: - project: Project to run BigQuery model prediction job. - location: Location to run the BigQuery model prediction job. If not set, default - to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for prediction. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#model_name - table_name: BigQuery table id of the input table that contains the data. For more - details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#table_name - query_statement: Query statement string used to generate - the data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#query_statement - contamination: Contamination is the proportion of anomalies in the training dataset - that are used to create the - AUTOENCODER, KMEANS, or PCA input models. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#contamination - anomaly_prob_threshold: The ARIMA_PLUS model supports the - anomaly_prob_threshold custom threshold for anomaly detection. The - value of the anomaly probability at each timestamp is calculated - using the actual time-series data value and the values of the - predicted time-series data and the variance from the model - training. The actual time-series data value at a specific - timestamp is identified as anomalous if the anomaly probability - exceeds the anomaly_prob_threshold value. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#anomaly_prob_threshold - query_parameters: Query parameters for standard SQL queries. - If query_parameters are both specified in here and in - job_configuration_query, the value in here will override the other one. - job_configuration_query: A json formatted string describing the rest of the job configuration. - For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery model prediction job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for prediction. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#model_name + table_name: BigQuery table id of the input table that contains the data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#table_name + query_statement: Query statement string used to generate the data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#query_statement + contamination: Contamination is the proportion of anomalies in the training dataset that are used to create the AUTOENCODER, KMEANS, or PCA input models. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#contamination + anomaly_prob_threshold: The ARIMA_PLUS model supports the anomaly_prob_threshold custom threshold for anomaly detection. The value of the anomaly probability at each timestamp is calculated using the actual time-series data value and the values of the predicted time-series data and the variance from the model training. The actual time-series data value at a specific timestamp is identified as anomalous if the anomaly probability exceeds the anomaly_prob_threshold value. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-detect-anomalies#anomaly_prob_threshold + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery model prediction job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the model prediction results should be - stored. - This property must be set for large results that exceed the maximum - response size. - For queries that produce anonymous (cached) results, this field will - be populated by BigQuery. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the model prediction results should be stored. This property must be set for large results that exceed the maximum response size. For queries that produce anonymous (cached) results, this field will be populated by BigQuery. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/drop_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/drop_model/component.py index 2b383e4e6a..359b1f1209 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/drop_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/drop_model/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component @@ -25,43 +26,28 @@ @container_component def bigquery_drop_model_job( - project: str, model: Input[BQMLModel], gcp_resources: OutputPath(str), location: str = 'us-central1', query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery drop model job and waits for it to finish. Args: - project: Project to run BigQuery model drop job. - location: Location of the job to drop the BigQuery - model. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location + location: Location of the job to drop the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location model: BigQuery ML model to drop. - query_parameters: Query parameters for standard SQL queries. - If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery model drop job. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/evaluate_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/evaluate_model/component.py index 91b6ecbc82..6a9cbb3a23 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/evaluate_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/evaluate_model/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_evaluate_model_job( - project: str, model: Input[BQMLModel], evaluation_metrics: Output[Artifact], gcp_resources: OutputPath(str), @@ -39,60 +39,27 @@ def bigquery_evaluate_model_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery evaluate model job and waits for it to finish. Args: - project: Project to run BigQuery model evaluation job. - location: Location to run the BigQuery model evaluation - job. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for evaluation. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_model_name - table_name: BigQuery table id of the input table that - contains the evaluation data, as in ML.EVALUATE(MODEL model_name[, - {TABLE table_name | (query_statement)}] For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_table_name - query_statement: Query statement string used to generate - the evaluation data, as in ML.EVALUATE(MODEL model_name[, {TABLE - table_name | (query_statement)}] For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_query_statement - threshold: A custom threshold for the binary-class - classification model to be used for evaluation. The default value is - 0.5. The threshold value that is supplied must be of type STRUCT. - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_threshold - query_parameters: jobs.query parameters for standard - SQL queries. If query_parameters are both specified in here and in - job_configuration_query, the value in here will override the other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can use - these to organize and group your jobs. Label keys and values can be no - longer than 63 characters, can only containlowercase letters, numeric - characters, underscores and dashes. International characters are - allowed. Label values are optional. Label keys must start with a letter - and each label in the list must have a different key. + location: Location to run the BigQuery model evaluation job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for evaluation. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_model_name + table_name: BigQuery table id of the input table that contains the evaluation data, as in ML.EVALUATE(MODEL model_name[, {TABLE table_name | (query_statement)}] For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_table_name + query_statement: Query statement string used to generate the evaluation data, as in ML.EVALUATE(MODEL model_name[, {TABLE table_name | (query_statement)}] For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_query_statement + threshold: A custom threshold for the binary-class classification model to be used for evaluation. The default value is 0.5. The threshold value that is supplied must be of type STRUCT. https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#eval_threshold + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud KMS - encryption key that will be used to protect destination BigQuery - table. The BigQuery Service Account associated with your project - requires access to this encryption key. If encryption_spec_key_name - are both specified in here and in job_configuration_query, the value - in here will override the other one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery model evaluation job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the model prediction results should be - stored. - This property must be set for large results that exceed the maximum - response size. - For queries that produce anonymous (cached) results, this field will - be populated by BigQuery. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the model prediction results should be stored. This property must be set for large results that exceed the maximum response size. For queries that produce anonymous (cached) results, this field will be populated by BigQuery. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_forecast_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_forecast_model/component.py index b0043c645b..cb3521abdd 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_forecast_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_forecast_model/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_explain_forecast_model_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -38,6 +38,7 @@ def bigquery_explain_forecast_model_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.EXPLAIN_FORECAST job and let you explain forecast an @@ -46,49 +47,20 @@ def bigquery_explain_forecast_model_job( This function only applies to the time-series ARIMA_PLUS and ARIMA models. Args: - project: Project to run the BigQuery job. - location: Location to run the BigQuery job. If not set, - default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for ML.EXPLAIN_FORECAST. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast - horizon: Horizon is the number of time points to explain forecast. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast#horizon - confidence_level: The percentage of the future values that fall in the prediction - interval. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast#confidence_level - query_parameters: Query parameters for standard SQL queries. If query_parameters are both - specified in here and in job_configuration_query, the value in here will - override the other one. - job_configuration_query: A json formatted string describing the rest of the job configuration. - For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.EXPLAIN_FORECAST. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast + horizon: Horizon is the number of time points to explain forecast. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast#horizon + confidence_level: The percentage of the future values that fall in the prediction interval. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast#confidence_level + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run the BigQuery job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the model explain forecast results should - be stored. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast#mlexplain_forecast_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the model explain forecast results should be stored. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-forecast#mlexplain_forecast_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_predict_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_predict_model/component.py index cb6e2f086b..5d5312c0fc 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_predict_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/explain_predict_model/component.py @@ -14,6 +14,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -26,7 +27,6 @@ @container_component def bigquery_explain_predict_model_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -40,72 +40,29 @@ def bigquery_explain_predict_model_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery explain predict model job and waits for it to finish. Args: - project: Project to run BigQuery model prediction job. - location: Location to run the BigQuery model prediction - job. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for explaining - prediction. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#model_name - table_name: BigQuery table id of the input table that - contains the prediction data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#table_name - query_statement: Query statement string used to generate - the prediction data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#query_statement - top_k_features: This argument specifies how many top - feature attribution pairs are generated per row of input data. The - features are ranked by the absolute values of their attributions. For - more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#top_k_features - threshold: A custom threshold for the binary logistic - regression model used as the cutoff between two labels. Predictions - above the threshold are treated as positive prediction. Predictions - below the threshold are negative predictions. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#threshold - num_integral_steps: This argument specifies the number - of steps to sample between the example being explained and its - baseline for approximating the integral in integrated gradients - attribution methods. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#num_integral_steps - query_parameters: Query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery model prediction job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for explaining prediction. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#model_name + table_name: BigQuery table id of the input table that contains the prediction data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#table_name + query_statement: Query statement string used to generate the prediction data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#query_statement + top_k_features: This argument specifies how many top feature attribution pairs are generated per row of input data. The features are ranked by the absolute values of their attributions. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#top_k_features + threshold: A custom threshold for the binary logistic regression model used as the cutoff between two labels. Predictions above the threshold are treated as positive prediction. Predictions below the threshold are negative predictions. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#threshold + num_integral_steps: This argument specifies the number of steps to sample between the example being explained and its baseline for approximating the integral in integrated gradients attribution methods. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-explain-predict#num_integral_steps + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery model prediction job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the model prediction results should be - stored. - This property must be set for large results that exceed the maximum - response size. - For queries that produce anonymous (cached) results, this field will - be populated by BigQuery. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the model prediction results should be stored. This property must be set for large results that exceed the maximum response size. For queries that produce anonymous (cached) results, this field will be populated by BigQuery. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/export_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/export_model/component.py index 032246d387..da56098be7 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/export_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/export_model/component.py @@ -15,6 +15,7 @@ from typing import Dict from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component @@ -25,7 +26,6 @@ @container_component def bigquery_export_model_job( - project: str, model: Input[BQMLModel], model_destination_path: str, exported_model_path: OutputPath(str), @@ -33,35 +33,23 @@ def bigquery_export_model_job( location: str = 'us-central1', job_configuration_extract: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery export model job and waits for it to finish. Args: - project: Project to run BigQuery model export job. - location: Location of the job to export the BigQuery - model. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location + location: Location of the job to export the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location model: BigQuery ML model to export. - model_destination_path: - The gcs bucket to export the - model to. - job_configuration_extract: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + model_destination_path: The gcs bucket to export the model to. + job_configuration_extract: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery model export job. Defaults to the project in which the PipelineJob is run. Returns: exported_model_path: The gcs bucket path where you export the model to. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/feature_importance/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/feature_importance/component.py index 446049504a..f6ae5a23a1 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/feature_importance/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/feature_importance/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_feature_importance_job( - project: str, model: Input[BQMLModel], feature_importance: Output[Artifact], gcp_resources: OutputPath(str), @@ -36,48 +36,25 @@ def bigquery_ml_feature_importance_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery feature importance fetching job and waits for it to finish. Args: - project: Project to run BigQuery model creation job. - location: Location of the job to create the BigQuery - model. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for feature - importance. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name - query_parameters: Query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location of the job to create the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for feature importance. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery model creation job. Defaults to the project in which the PipelineJob is run. + Returns: - feature_importance: Describes common metrics applicable to the type of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-importance - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + feature_importance: Describes common metrics applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-importance + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/forecast_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/forecast_model/component.py index 2e8363b5a4..c94d3aa3dc 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/forecast_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/forecast_model/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_forecast_model_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -38,6 +38,7 @@ def bigquery_forecast_model_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.FORECAST job and let you forecast an ARIMA_PLUS or @@ -46,48 +47,21 @@ def bigquery_forecast_model_job( This function only applies to the time-series ARIMA_PLUS and ARIMA models. Args: - project: Project to run the BigQuery job. - location: Location to run the BigQuery job. If not set, - default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for ML.FORECAST. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast - horizon: Horizon is the number of time points to - forecast. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast#horizon - confidence_level: The percentage of the future values - that fall in the prediction interval. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast#confidence_level - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.FORECAST. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast + horizon: Horizon is the number of time points to forecast. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast#horizon + confidence_level: The percentage of the future values that fall in the prediction interval. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast#confidence_level + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: Describes the Cloud KMS - encryption key that will be used to protect destination BigQuery - table. The BigQuery Service Account associated with your project - requires access to this encryption key. If encryption_spec_key_name - are both specified in here and in job_configuration_query, the value - in here will override the other one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run the BigQuery job. Defaults to the project in which the PipelineJob is run. + Returns: - destination_table: Describes the table where the model forecast results should be - stored. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast#mlforecast_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the model forecast results should be stored. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-forecast#mlforecast_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/global_explain/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/global_explain/component.py index 6e7c55ef3a..dbd64a7e14 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/global_explain/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/global_explain/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_global_explain_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -37,31 +37,20 @@ def bigquery_ml_global_explain_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery global explain fetching job and waits for it to finish. Args: - project: Project to run BigQuery model creation job. - location: Location of the job to create the BigQuery - model. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for global - explain. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name - class_level_explain: For classification - models, if class_level_explain is set to TRUE then global feature - importances are returned for each class. Otherwise, the global - feature importance of the entire model is returned rather than that - of each class. By default, class_level_explain is set to FALSE. This - option only applies to classification models. Regression models only - have model-level global feature importance. + location: Location of the job to create the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for global explain. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name + class_level_explain: For classification models, if class_level_explain is set to TRUE then global feature importances are returned for each class. Otherwise, the global feature importance of the entire model is returned rather than that of each class. By default, class_level_explain is set to FALSE. This option only applies to classification models. Regression models only have model-level global feature importance. + project: Project to run BigQuery model creation job. Defaults to the project in which the PipelineJob is run. Returns: destination_table: Describes the table where the global explain results should be stored. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_advanced_weights/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_advanced_weights/component.py index f6a9536d32..a51e414d10 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_advanced_weights/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_advanced_weights/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_advanced_weights_job( - project: str, model: Input[BQMLModel], advanced_weights: Output[Artifact], gcp_resources: OutputPath(str), @@ -35,41 +35,23 @@ def bigquery_ml_advanced_weights_job( query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ml advanced weights job and waits for it to finish. Args: - project: Project to run BigQuery ml advanced weights job. - location: Location of the job to create the BigQuery - model. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for ml advanced - weights job. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location of the job to create the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ml advanced weights job. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery ml advanced weights job. Defaults to the project in which the PipelineJob is run. Returns: - weights: Describes different output columns for different models. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-advanced-weights#mladvanced_weights_output. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + weights: Describes different output columns for different models. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-advanced-weights#mladvanced_weights_output. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_coefficients/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_coefficients/component.py index 60f817b277..748c1547b6 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_coefficients/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_coefficients/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_arima_coefficients( - project: str, model: Input[BQMLModel], arima_coefficients: Output[Artifact], gcp_resources: OutputPath(str), @@ -36,6 +36,7 @@ def bigquery_ml_arima_coefficients( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.ARIMA_COEFFICIENTS job and let you see the ARIMA @@ -44,36 +45,16 @@ def bigquery_ml_arima_coefficients( This function only applies to the time-series ARIMA_PLUS and ARIMA models. Args: - project: Project to run the BigQuery job. - location: Location to run the BigQuery job. If not set, - default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for - ML.ARIMA_COEFFICIENTS. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-coefficients - query_parameters: Query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + location: Location to run the BigQuery job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.ARIMA_COEFFICIENTS. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-coefficients + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run the BigQuery job. Defaults to the project in which the PipelineJob is run. Returns: - arima_coefficients: Describes arima_coefficients to the type of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-coefficients#mlarima_coefficients_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + arima_coefficients: Describes arima_coefficients to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-coefficients#mlarima_coefficients_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_evaluate/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_evaluate/component.py index 519d60db1d..841a095966 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_evaluate/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_arima_evaluate/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_arima_evaluate_job( - project: str, model: Input[BQMLModel], arima_evaluation_metrics: Output[Artifact], gcp_resources: OutputPath(str), @@ -37,54 +37,25 @@ def bigquery_ml_arima_evaluate_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.ARIMA_EVALUATE job and waits for it to finish. Args: - project: Project to run BigQuery model evaluation job. - location: Location to run the BigQuery model evaluation - job. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for - ML.ARIMA_EVALUATE. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-evaluate#model_name - show_all_candidate_models: You can use - show_all_candidate_models to show evaluation metrics or an error - message for either all candidate models or for only the best model - with the lowest AIC. The value is type BOOL and is part of the - settings STRUCT. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-evaluate#show_all_candidate_models - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery model evaluation job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.ARIMA_EVALUATE. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-evaluate#model_name + show_all_candidate_models: You can use show_all_candidate_models to show evaluation metrics or an error message for either all candidate models or for only the best model with the lowest AIC. The value is type BOOL and is part of the settings STRUCT. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-evaluate#show_all_candidate_models + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery model evaluation job. Defaults to the project in which the PipelineJob is run. Returns: - arima_evaluation_metrics: Describes arima metrics. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-evaluate#mlarima_evaluate_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + arima_evaluation_metrics: Describes arima metrics. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-arima-evaluate#mlarima_evaluate_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_centroids/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_centroids/component.py index 62dfaa6a63..5c30d8f1d7 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_centroids/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_centroids/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_centroids_job( - project: str, model: Input[BQMLModel], centroids: Output[Artifact], gcp_resources: OutputPath(str), @@ -37,52 +37,25 @@ def bigquery_ml_centroids_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.CENTROIDS job and waits for it to finish. Args: - project: Project to run BigQuery ML.CENTROIDS job. - location: Location to run the BigQuery ML.CENTROIDS job. - If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for ML.CENTROIDS. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-centroids#mlcentroids_syntax - standardize: Determines whether the centroid features - should be standardized to assume that all features have a mean of zero - and a standard deviation of one. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-centroids#mlcentroids_syntax - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery ML.CENTROIDS job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.CENTROIDS. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-centroids#mlcentroids_syntax + standardize: Determines whether the centroid features should be standardized to assume that all features have a mean of zero and a standard deviation of one. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-centroids#mlcentroids_syntax + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery ML.CENTROIDS job. Defaults to the project in which the PipelineJob is run. Returns: - centroids: Information about the centroids in a k-means model. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-centroids#mlcentroids_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + centroids: Information about the centroids in a k-means model. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-centroids#mlcentroids_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_confusion_matrix/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_confusion_matrix/component.py index 8005826b64..7ce55a27d7 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_confusion_matrix/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_confusion_matrix/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_confusion_matrix_job( - project: str, model: Input[BQMLModel], confusion_matrix: Output[BQTable], gcp_resources: OutputPath(str), @@ -38,49 +38,26 @@ def bigquery_ml_confusion_matrix_job( query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery confusion matrix job and waits for it to finish. Args: - project: Project to run BigQuery confusion matrix job. - location: Location to run the BigQuery confusion matrix - job. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for confusion - matrix. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_model_name - table_name: BigQuery table id of the input table that - contains the evaluation data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_table_name - query_statement: Query statement string used to generate - the evaluation data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_query_statement - threshold: A custom threshold for your binary - classification model used for evaluation. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_threshold - query_parameters: Query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery confusion matrix job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for confusion matrix. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_model_name + table_name: BigQuery table id of the input table that contains the evaluation data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_table_name + query_statement: Query statement string used to generate the evaluation data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_query_statement + threshold: A custom threshold for your binary classification model used for evaluation. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#eval_threshold + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery confusion matrix job. Defaults to the project in which the PipelineJob is run. Returns: - confusion_matrix: Describes common metrics applicable to the type of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#mlconfusion_matrix_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + confusion_matrix: Describes common metrics applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-confusion#mlconfusion_matrix_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_feature_info/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_feature_info/component.py index d4e2dfd3bb..4ce9884154 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_feature_info/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_feature_info/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_feature_info_job( - project: str, model: Input[BQMLModel], feature_info: Output[Artifact], gcp_resources: OutputPath(str), @@ -35,41 +35,23 @@ def bigquery_ml_feature_info_job( query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery feature info job and waits for it to finish. Args: - project: Project to run BigQuery feature info job. - location: Location of the job to run BigQuery feature - info job. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for evaluation. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location of the job to run BigQuery feature info job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for evaluation. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery feature info job. Defaults to the project in which the PipelineJob is run. Returns: - feature_info: Describes common metrics applicable to the type of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-feature#mlfeature_info_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + feature_info: Describes common metrics applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-feature#mlfeature_info_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_component_info/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_component_info/component.py index ad0032f43f..505b301a6a 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_component_info/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_component_info/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_principal_component_info_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -36,52 +36,25 @@ def bigquery_ml_principal_component_info_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.principal_component_info job and waits for it to finish. Args: - project: Project to run BigQuery - ML.principal_component_info job. - location: Location to run the BigQuery - ML.principal_component_info job. If not set, default to `US` - multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for - ML.principal_component_info. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-component-info#mlprincipal_component_info_syntax - query_parameters: Query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery ML.principal_component_info job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.principal_component_info. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-component-info#mlprincipal_component_info_syntax + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery ML.principal_component_info job. Defaults to the project in which PipelineJob is run. Returns: - destination_table: Describes the table which stores common metrics applicable to the type - of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-component-info#mlprincipal_component_info_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table which stores common metrics applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-component-info#mlprincipal_component_info_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_components/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_components/component.py index aafae8803d..a3b7ede1c8 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_components/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_principal_components/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_principal_components_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -36,49 +36,24 @@ def bigquery_ml_principal_components_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.principal_components job and waits for it to finish. Args: - project: Project to run BigQuery ML.principal_components - job. - location: Location to run the BigQuery - ML.principal_components job. If not set, default to `US` multi-region. - For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for - ML.principal_components. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-components#mlprincipal_components_syntax - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery ML.principal_components job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.principal_components. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-components#mlprincipal_components_syntax + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: Describes the Cloud KMS - encryption key that will be used to protect destination BigQuery - table. The BigQuery Service Account associated with your project - requires access to this encryption key. If encryption_spec_key_name - are both specified in here and in job_configuration_query, the value - in here will override the other one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery ML.principal_components job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table which stores common metrics applicable to the type - of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-components#mlprincipal_components_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table which stores common metrics applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-principal-components#mlprincipal_components_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_recommend/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_recommend/component.py index 2fd7a6e1b0..cda1362b0f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_recommend/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_recommend/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_recommend_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -38,53 +38,26 @@ def bigquery_ml_recommend_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ML.Recommend job and waits for it to finish. Args: - project: Project to run BigQuery ML.Recommend job. - location: Location to run the BigQuery ML.Recommend job. - If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for ML.Recoomend. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-recommend#recommend_model_name - table_name: BigQuery table id of the input table that - contains the the user and/or item data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-recommend#recommend_table_name - query_statement: query statement string used to generate - the evaluation data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-recommend#recommend_query_statement - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery ML.Recommend job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for ML.Recoomend. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-recommend#recommend_model_name + table_name: BigQuery table id of the input table that contains the the user and/or item data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-recommend#recommend_table_name + query_statement: query statement string used to generate the evaluation data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-recommend#recommend_query_statement + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: Describes the Cloud KMS - encryption key that will be used to protect destination BigQuery - table. The BigQuery Service Account associated with your project - requires access to this encryption key. If encryption_spec_key_name - are both specified in here and in job_configuration_query, the value - in here will override the other one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery ML.Recommend job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the recommendation results should be stored. - This property must be set for large results that exceed the maximum - response size. - For queries that produce anonymous (cached) results, this field will - be populated by BigQuery. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the recommendation results should be stored. This property must be set for large results that exceed the maximum response size. For queries that produce anonymous (cached) results, this field will be populated by BigQuery. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_reconstruction_loss/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_reconstruction_loss/component.py index dfaa6b08ff..4a80cfb88e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_reconstruction_loss/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_reconstruction_loss/component.py @@ -14,6 +14,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -26,7 +27,6 @@ @container_component def bigquery_ml_reconstruction_loss_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -37,54 +37,26 @@ def bigquery_ml_reconstruction_loss_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ml reconstruction loss job and waits for it to finish. Args: - project: Project to run BigQuery ml reconstruction loss - job. - location: Location to run the BigQuery ml reconstruction - loss job. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model. For more details, - see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-reconstruction-loss#reconstruction_loss_model_name - table_name: BigQuery table id of the input table that - contains the input data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-reconstruction-loss#reconstruction_loss_table_name - query_statement: Query statement string used to generate - the input data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-reconstruction-loss#reconstruction_loss_query_statement - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery ml reconstruction loss job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-reconstruction-loss#reconstruction_loss_model_name + table_name: BigQuery table id of the input table that contains the input data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-reconstruction-loss#reconstruction_loss_table_name + query_statement: Query statement string used to generate the input data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-reconstruction-loss#reconstruction_loss_query_statement + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: Describes the Cloud KMS - encryption key that will be used to protect destination BigQuery - table. The BigQuery Service Account associated with your project - requires access to this encryption key. If encryption_spec_key_name - are both specified in here and in job_configuration_query, the value - in here will override the other one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery ml reconstruction loss job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the ml reconstruction loss job results - should be stored. This property must be set for large results that - exceed the maximum response size. For queries that produce anonymous - (cached) results, this field will be populated by BigQuery. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the ml reconstruction loss job results should be stored. This property must be set for large results that exceed the maximum response size. For queries that produce anonymous (cached) results, this field will be populated by BigQuery. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_roc_curve/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_roc_curve/component.py index 4e1de3db2d..c7a7f2f841 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_roc_curve/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_roc_curve/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_roc_curve_job( - project: str, model: Input[BQMLModel], roc_curve: Output[BQTable], gcp_resources: OutputPath(str), @@ -38,49 +38,26 @@ def bigquery_ml_roc_curve_job( query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery roc curve job and waits for it to finish. Args: - project: Project to run BigQuery roc curve job. - location: Location of the job to run BigQuery roc curve - job. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for BigQuery roc - curv job. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_model_name - table_name: BigQuery table id of the input table that - contains the evaluation data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_table_name - query_statement: Query statement string used to generate - the evaluation data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_query_statement - thresholds: Percentile values of the prediction output. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_thresholds - query_parameters: Query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location of the job to run BigQuery roc curve job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for BigQuery roc curv job. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_model_name + table_name: BigQuery table id of the input table that contains the evaluation data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_table_name + query_statement: Query statement string used to generate the evaluation data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_query_statement + thresholds: Percentile values of the prediction output. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#roc_thresholds + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery roc curve job. Defaults to the project in which the PipelineJob is run. Returns: - roc_curve: Describes common metrics applicable to the type of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#mlroc_curve_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + roc_curve: Describes common metrics applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-roc#mlroc_curve_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_training_info/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_training_info/component.py index bb8695d666..4cb871769b 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_training_info/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_training_info/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_training_info_job( - project: str, model: Input[BQMLModel], ml_training_info: Output[Artifact], gcp_resources: OutputPath(str), @@ -35,43 +35,24 @@ def bigquery_ml_training_info_job( query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ml training info fetching job and waits for it to finish. Args: - project: Project to run BigQuery ML training info job. - location: Location of the job to create the BigQuery - model. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - query: SQL query text to execute. Only standard SQL is - supported. If query are both specified in here and in - job_configuration_query, the value in here will override the other - one. - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location of the job to create the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + query: SQL query text to execute. Only standard SQL is supported. If query are both specified in here and in job_configuration_query, the value in here will override the other one. + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery ML training info job. Defaults to the project in which the PipelineJob is run. Returns: - ml_training_info: Describes common metrics applicable to the type of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + ml_training_info: Describes common metrics applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate#mlevaluate_output + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_trial_info/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_trial_info/component.py index 5adbe7f5cf..87e61cfc90 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_trial_info/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_trial_info/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_trial_info_job( - project: str, model: Input[BQMLModel], trial_info: Output[Artifact], gcp_resources: OutputPath(str), @@ -36,45 +36,24 @@ def bigquery_ml_trial_info_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ml trial info job and waits for it to finish. Args: - project: Project to run BigQuery ml trial info job. - location: Location to run the BigQuery ml trial info - job. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model. For more details, - see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-trial-info#predict_model_name - query_parameters: Query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery ml trial info job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-trial-info#predict_model_name + query_parameters: Query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: Describes the Cloud KMS - encryption key that will be used to protect destination BigQuery - table. The BigQuery Service Account associated with your project - requires access to this encryption key. If encryption_spec_key_name - are both specified in here and in job_configuration_query, the value - in here will override the other one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery ml trial info job. Defaults to the project in which the PipelineJob is run. Returns: - trial_info: Describes the trial info applicable to the type of model supplied. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-trial-info - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + trial_info: Describes the trial info applicable to the type of model supplied. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-trial-info + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_weights/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_weights/component.py index 4f799e8edf..10841021a2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_weights/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/ml_weights/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from kfp.dsl import Artifact from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_ml_weights_job( - project: str, model: Input[BQMLModel], weights: Output[Artifact], gcp_resources: OutputPath(str), @@ -35,42 +35,23 @@ def bigquery_ml_weights_job( query_parameters: List[str] = [], job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery ml weights job and waits for it to finish. Args: - project: Project to run BigQuery ml weights job. - location: Location of the job to create the BigQuery - model. If not set, default to `US` multi-region. For more details, - see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - query: SQL query text to execute. Only standard SQL is - supported. If query are both specified in here and in - job_configuration_query, the value in here will override the other - one. - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location of the job to create the BigQuery model. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + query: SQL query text to execute. Only standard SQL is supported. If query are both specified in here and in job_configuration_query, the value in here will override the other one. + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. + project: Project to run BigQuery ml weights job. Defaults to the project in which the PipelineJob is run. Returns: - weights: Describes different output columns for different models. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-weights#mlweights_output. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + weights: Describes different output columns for different models. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-weights#mlweights_output. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/predict_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/predict_model/component.py index fa912330dc..8fbb21d354 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/predict_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/predict_model/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQMLModel from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder @@ -27,7 +28,6 @@ @container_component def bigquery_predict_model_job( - project: str, model: Input[BQMLModel], destination_table: Output[BQTable], gcp_resources: OutputPath(str), @@ -39,60 +39,27 @@ def bigquery_predict_model_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery predict model job and waits for it to finish. Args: - project: Project to run BigQuery model prediction job. - location: Location to run the BigQuery model prediction - job. If not set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - model: BigQuery ML model for prediction. - For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name - table_name: BigQuery table id of the input table that - contains the prediction data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_table_name - query_statement: Query statement string used to generate - the prediction data. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_query_statement - threshold: A custom threshold for the binary logistic - regression model used as the cutoff between two labels. Predictions - above the threshold are treated as positive prediction. Predictions - below the threshold are negative predictions. For more details, see - https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#threshold - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location to run the BigQuery model prediction job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + model: BigQuery ML model for prediction. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_model_name + table_name: BigQuery table id of the input table that contains the prediction data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_table_name + query_statement: Query statement string used to generate the prediction data. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#predict_query_statement + threshold: A custom threshold for the binary logistic regression model used as the cutoff between two labels. Predictions above the threshold are treated as positive prediction. Predictions below the threshold are negative predictions. For more details, see https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict#threshold + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: Describes the Cloud KMS - encryption key that will be used to protect destination BigQuery - table. The BigQuery Service Account associated with your project - requires access to this encryption key. If encryption_spec_key_name - are both specified in here and in job_configuration_query, the value - in here will override the other one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run BigQuery model prediction job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the model prediction results should be - stored. - This property must be set for large results that exceed the maximum - response size. - For queries that produce anonymous (cached) results, this field will - be populated by BigQuery. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the model prediction results should be stored. This property must be set for large results that exceed the maximum response size. For queries that produce anonymous (cached) results, this field will be populated by BigQuery. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/query_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/query_job/component.py index 4daa7cd03a..625330844d 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/query_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/bigquery/query_job/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import BQTable from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component @@ -25,7 +26,6 @@ @container_component def bigquery_query_job( - project: str, destination_table: Output[BQTable], gcp_resources: OutputPath(str), query: str = '', @@ -34,51 +34,24 @@ def bigquery_query_job( job_configuration_query: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a BigQuery query job and waits for it to finish. Args: - project: Project to run the BigQuery query job. - location: Location for creating the BigQuery job. If not - set, default to `US` multi-region. For more details, see - https://cloud.google.com/bigquery/docs/locations#specifying_your_location - query: SQL query text to execute. Only standard SQL is - supported. If query are both specified in here and in - job_configuration_query, the value in here will override the other - one. - query_parameters: jobs.query parameters for - standard SQL queries. If query_parameters are both specified in here - and in job_configuration_query, the value in here will override the - other one. - job_configuration_query: A json formatted string - describing the rest of the job configuration. For more details, see - https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery - labels: The labels associated with this job. You can - use these to organize and group your jobs. Label keys and values can - be no longer than 63 characters, can only containlowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. Label values are optional. Label keys must start with a - letter and each label in the list must have a different key. + location: Location for creating the BigQuery job. If not set, default to `US` multi-region. For more details, see https://cloud.google.com/bigquery/docs/locations#specifying_your_location + query: SQL query text to execute. Only standard SQL is supported. If query are both specified in here and in job_configuration_query, the value in here will override the other one. + query_parameters: jobs.query parameters for standard SQL queries. If query_parameters are both specified in here and in job_configuration_query, the value in here will override the other one. + job_configuration_query: A json formatted string describing the rest of the job configuration. For more details, see https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery + labels: The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only containlowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key. Example: { "name": "wrench", "mass": "1.3kg", "count": "3" }. - encryption_spec_key_name: - Describes the Cloud - KMS encryption key that will be used to protect destination - BigQuery table. The BigQuery Service Account associated with your - project requires access to this encryption key. If - encryption_spec_key_name are both specified in here and in - job_configuration_query, the value in here will override the other - one. + encryption_spec_key_name: Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key. If encryption_spec_key_name are both specified in here and in job_configuration_query, the value in here will override the other one. + project: Project to run the BigQuery query job. Defaults to the project in which the PipelineJob is run. Returns: - destination_table: Describes the table where the query results should be stored. - This property must be set for large results that exceed the maximum - response size. - For queries that produce anonymous (cached) results, this field will - be populated by BigQuery. - gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. - For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + destination_table: Describes the table where the query results should be stored. This property must be set for large results that exceed the maximum response size. For queries that produce anonymous (cached) results, this field will be populated by BigQuery. + gcp_resources: Serialized gcp_resources proto tracking the BigQuery job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/__init__.py index 6fa091caf6..0075f3f221 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/__init__.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Run KFP components as `Vertex AI Custom Training Jobs `_ with customized worker and cloud configurations.""" +# fmt: off +"""Run KFP components as [Vertex AI Custom Training Jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) with customized worker and cloud configurations.""" +# fmt: on from google_cloud_pipeline_components.v1.custom_job.component import custom_training_job as CustomTrainingJobOp -from google_cloud_pipeline_components.v1.custom_job.utils import ( - create_custom_training_job_from_component, - create_custom_training_job_op_from_component, -) +from google_cloud_pipeline_components.v1.custom_job.utils import create_custom_training_job_from_component +from google_cloud_pipeline_components.v1.custom_job.utils import create_custom_training_job_op_from_component __all__ = [ 'CustomTrainingJobOp', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/component.py index 2667f04c87..5134a5e4d3 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/component.py @@ -14,6 +14,7 @@ from typing import Dict, List +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components import utils from kfp import dsl @@ -21,7 +22,6 @@ # keep identical to create_custom_training_job_from_component @dsl.container_component def custom_training_job( - project: str, display_name: str, gcp_resources: dsl.OutputPath(str), location: str = 'us-central1', @@ -36,64 +36,29 @@ def custom_training_job( base_output_directory: str = '', labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Launch a Vertex AI `custom training job `_ using the `CustomJob `_ API. - - See `Create custom training jobs - `_ for - more information. + """Launch a Vertex AI [custom training job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) using the [CustomJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.customJobs) API. See [Create custom training jobs ](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) for more information. Args: - project: Project to create the custom training job in. - location: Location for creating the custom training job. - If not set, default to us-central1. + location: Location for creating the custom training job. If not set, default to us-central1. display_name: The name of the CustomJob. - worker_pool_specs: Serialized json spec of the - worker pools including machine type and Docker image. All worker pools - except the first one are optional and can be skipped by providing an - empty value. See `more information `_. - timeout: The maximum job running time. The default is 7 days. A duration in - seconds with up to nine fractional digits, terminated by 's', for example: - "3.5s". - restart_job_on_worker_restart: Restarts the entire CustomJob if a worker - gets restarted. This feature can be used by distributed training jobs that - are not resilient to workers leaving and joining a job. - service_account: Sets the default service account for workload run-as - account. The `service account - `_ - running the pipeline submitting jobs must have act-as permission on this - run-as account. If unspecified, the Vertex AI Custom Code `Service Agent - `_ - for the CustomJob's project. - tensorboard: The name of a Vertex AI Tensorboard resource to which this - CustomJob will upload Tensorboard logs. - enable_web_access: Whether you want Vertex AI to enable `interactive shell - access - `_ - to training containers. If ``True``, you can access interactive shells at - the URIs given by [CustomJob.web_access_uris][]. - network: The full name of the Compute Engine network to which the job should - be peered. For example, ``projects/12345/global/networks/myVPC``. Format - is of the form ``projects/{project}/global/networks/{network}``. Where - ``{project}`` is a project number, as in ``12345``, and ``{network}`` is a - network name. Private services access must already be configured for the - network. If left unspecified, the job is not peered with any network. - reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC - network that can be used for this job. If set, we will deploy the job - within the provided IP ranges. Otherwise, the job will be deployed to any - IP ranges under the provided VPC network. - base_output_directory: The Cloud Storage location to store the output of - this CustomJob or HyperparameterTuningJob. See `more information - `_. - labels: The labels with user-defined metadata to organize the CustomJob. See - `more information `_. - encryption_spec_key_name: Customer-managed encryption key options for the - CustomJob. If this is set, then all resources created by the CustomJob - will be encrypted with the provided encryption key. + worker_pool_specs: Serialized json spec of the worker pools including machine type and Docker image. All worker pools except the first one are optional and can be skipped by providing an empty value. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec#WorkerPoolSpec). + timeout: The maximum job running time. The default is 7 days. A duration in seconds with up to nine fractional digits, terminated by 's', for example: "3.5s". + restart_job_on_worker_restart: Restarts the entire CustomJob if a worker gets restarted. This feature can be used by distributed training jobs that are not resilient to workers leaving and joining a job. + service_account: Sets the default service account for workload run-as account. The [service account ](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) running the pipeline submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code [Service Agent ](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. + tensorboard: The name of a Vertex AI TensorBoard resource to which this CustomJob will upload TensorBoard logs. + enable_web_access: Whether you want Vertex AI to enable [interactive shell access ](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) to training containers. If `True`, you can access interactive shells at the URIs given by [CustomJob.web_access_uris][]. + network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name. Private services access must already be configured for the network. If left unspecified, the job is not peered with any network. + reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC network that can be used for this job. If set, we will deploy the job within the provided IP ranges. Otherwise, the job will be deployed to any IP ranges under the provided VPC network. + base_output_directory: The Cloud Storage location to store the output of this CustomJob or HyperparameterTuningJob. See [more information ](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/GcsDestination). + labels: The labels with user-defined metadata to organize the CustomJob. See [more information](https://goo.gl/xmQnxf). + encryption_spec_key_name: Customer-managed encryption key options for the CustomJob. If this is set, then all resources created by the CustomJob will be encrypted with the provided encryption key. + project: Project to create the custom training job in. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the CustomJob. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the CustomJob. """ # fmt: on return utils.build_serverless_customjob_container_spec( @@ -119,9 +84,7 @@ def custom_training_job( }, }, 'labels': labels, - 'encryption_spec_key_name': { - 'kms_key_name': encryption_spec_key_name - }, + 'encryption_spec': {'kms_key_name': encryption_spec_key_name}, }, gcp_resources=gcp_resources, ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_info.textproto b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_info.textproto deleted file mode 100644 index 296accb6b6..0000000000 --- a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_info.textproto +++ /dev/null @@ -1,30 +0,0 @@ -# proto-file: third_party/py/google_cloud_pipeline_components/templates_in_gallery/protos/gallery_metadata.proto -# proto-message: GalleryMetadata - -# Forked from google3/third_party/py/google_cloud_pipeline_components/templates_in_gallery/protos/gallery_metadata_sample.textproto -upload_template_request { - vertex_template_gallery_metadata { - # Refer to - # https://source.corp.google.com/piper///depot/google3/google/devtools/artifactregistry/main/kfp_artifact.proto;rcl=536882057;l=27 - # for the list of options. - # - # And if you would like to propose some new categorization or types, please review - # https://docs.google.com/spreadsheets/d/16em2Dp-sHpJW61rP8SiItyty2Dor4DquDORQEIWyo-0/edit?resourcekey=0-pMwjYRX_DwHd1U0lRdEWEA#gid=0 - # and contact @desmliu - vertex_gallery_categorization { - type: COMPONENT - integration: VERTEX_AI - } - # This field is a temporary solution. Will remove shortly after b/285601340. - # Please add display name in the python file, example of - # http://google3/third_party/py/google_cloud_pipeline_components/google_cloud_pipeline_components/google/template_in_gallery_test/python/component.py;rcl=529459116. - # TODO(b/285601340) remove this field. - display_name: "Vertex AI Custom Job Component" - } - # This field is a temporary solution. Will remove shortly after b/285601340. - # Please add description in the python file, example of - # http://google3/third_party/py/google_cloud_pipeline_components/google_cloud_pipeline_components/google/template_in_gallery_test/python/component.py;rcl=529459116. - # TODO(b/285601340) remove this field. - description: "Launch a Custom training job using Vertex CustomJob API." -} -pipeline_func: "google_cloud_pipeline_components.v1.custom_job.CustomTrainingJobOp" diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_reference_doc.md b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_reference_doc.md deleted file mode 100644 index 61b8cbb971..0000000000 --- a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_reference_doc.md +++ /dev/null @@ -1,3 +0,0 @@ -Custom training jobs let you run your custom machine learning (ML) training code in Vertex AI. - -Refer to the [User Doc](https://cloud.google.com/vertex-ai/docs/pipelines/customjob-component) \ No newline at end of file diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_release_note.md b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_release_note.md deleted file mode 100644 index 32f5ad1478..0000000000 --- a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/template_in_gallery/component_release_note.md +++ /dev/null @@ -1,3 +0,0 @@ -2023-05-18 - -Release to Vertex AI Pipelines Template Gallery. \ No newline at end of file diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/utils.py b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/utils.py index 9182c85e80..e83d374175 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/utils.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/custom_job/utils.py @@ -33,8 +33,7 @@ def _replace_executor_placeholder( Args: container_input: Container command or args. - Returns: - container_input with executor placeholder replaced. + Returns: container_input with executor placeholder replaced. """ # Executor replacement is used as executor content needs to be jsonified before # injection into the payload, since payload is already a JSON serialized string. @@ -70,98 +69,36 @@ def create_custom_training_job_from_component( base_output_directory: str = '', labels: Optional[Dict[str, str]] = None, ) -> Callable: - """Convert a KFP component into Vertex AI `custom training job `_ using the `CustomJob `_ API. - - This utility converts a `KFP component - `_ - provided to ``component_spec`` into ``CustomTrainingJobOp`` component. Your - components inputs, outputs, and logic are carried over, with additional - `CustomJob - `_ - parameters exposed. - - Note that this utility constructs a ClusterSpec where the master and all the - workers use the same spec, meaning all disk/machine spec related parameters - will apply to all replicas. This is suitable for uses cases such as executing - a training component over multiple replicas with `MultiWorkerMirroredStrategy - `_ - or `MirroredStrategy - `_. - - See `Create custom training jobs - `_ for - more information. - - Args: - component_spec: A KFP component. - display_name: The name of the CustomJob. If not provided the component's - name will be used instead. - replica_count: The count of instances in the cluster. One replica always - counts towards the master in worker_pool_spec[0] and the remaining - replicas will be allocated in worker_pool_spec[1]. See `more information. - `_ - machine_type: The type of the machine to run the CustomJob. The default - value is "n1-standard-4". See `more information - `_. - accelerator_type: The type of accelerator(s) that may be attached to the - machine per ``accelerator_count``. See `more information - `_. - accelerator_count: The number of accelerators to attach to the machine. - Defaults to 1 if ``accelerator_type`` is set. - boot_disk_type: Type of the boot disk (default is "pd-ssd"). Valid values: - "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent - Disk Hard Disk Drive). boot_disk_type is set as a static value and cannot - be changed as a pipeline parameter. - boot_disk_size_gb: Size in GB of the boot disk (default is 100GB). - ``boot_disk_size_gb`` is set as a static value and cannot be changed as a - pipeline parameter. - timeout: The maximum job running time. The default is 7 days. A duration in - seconds with up to nine fractional digits, terminated by 's', for example: - "3.5s". - restart_job_on_worker_restart: Restarts the entire CustomJob if a worker - gets restarted. This feature can be used by distributed training jobs that - are not resilient to workers leaving and joining a job. - service_account: Sets the default service account for workload run-as - account. The `service account - `_ - running the pipeline submitting jobs must have act-as permission on this - run-as account. If unspecified, the Vertex AI Custom Code `Service Agent - `_ - for the CustomJob's project. - network: The full name of the Compute Engine network to which the job should - be peered. For example, ``projects/12345/global/networks/myVPC``. Format - is of the form ``projects/{project}/global/networks/{network}``. Where - ``{project}`` is a project number, as in ``12345``, and ``{network}`` is a - network name. Private services access must already be configured for the - network. If left unspecified, the job is not peered with any network. - encryption_spec_key_name: Customer-managed encryption key options for the - CustomJob. If this is set, then all resources created by the CustomJob - will be encrypted with the provided encryption key. - tensorboard: The name of a Vertex AI Tensorboard resource to which this - CustomJob will upload Tensorboard logs. - enable_web_access: Whether you want Vertex AI to enable `interactive shell - access - `_ - to training containers. If ``True``, you can access interactive shells at - the URIs given by [CustomJob.web_access_uris][]. - reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC - network that can be used for this job. If set, we will deploy the job - within the provided IP ranges. Otherwise, the job will be deployed to any - IP ranges under the provided VPC network. - nfs_mounts: A list of `NfsMount - `_ - resource specs in Json dict format. For more details about mounting NFS - for CustomJob, see `Mount an NFS share for custom training - `_. - base_output_directory: The Cloud Storage location to store the output of - this CustomJob or HyperparameterTuningJob. See `more information - `_. - labels: The labels with user-defined metadata to organize the CustomJob. See - `more information `_. - - Returns: - A KFP component with CustomJob specification applied. + # fmt: off + """Convert a KFP component into Vertex AI [custom training job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) using the [CustomJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.customJobs) API. + + This utility converts a [KFP component](https://www.kubeflow.org/docs/components/pipelines/v2/components/) provided to `component_spec` into `CustomTrainingJobOp` component. Your components inputs, outputs, and logic are carried over, with additional [CustomJob](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec) parameters exposed. Note that this utility constructs a ClusterSpec where the master and all the workers use the same spec, meaning all disk/machine spec related parameters will apply to all replicas. This is suitable for uses cases such as executing a training component over multiple replicas with [MultiWorkerMirroredStrategy](https://www.tensorflow.org/api_docs/python/tf/distribute/MultiWorkerMirroredStrategy) or [MirroredStrategy](https://www.tensorflow.org/api_docs/python/tf/distribute/MirroredStrategy). See [Create custom training jobs](https://cloud.google.com/vertex-ai/docs/training/create-custom-job) for more information. + + Args: + component_spec: A KFP component. + display_name: The name of the CustomJob. If not provided the component's name will be used instead. + replica_count: The count of instances in the cluster. One replica always counts towards the master in worker_pool_spec[0] and the remaining replicas will be allocated in worker_pool_spec[1]. See [more information.](https://cloud.google.com/vertex-ai/docs/training/distributed-training#configure_a_distributed_training_job) + machine_type: The type of the machine to run the CustomJob. The default value is "n1-standard-4". See [more information](https://cloud.google.com/vertex-ai/docs/training/configure-compute#machine-types). + accelerator_type: The type of accelerator(s) that may be attached to the machine per `accelerator_count`. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec#acceleratortype). + accelerator_count: The number of accelerators to attach to the machine. Defaults to 1 if `accelerator_type` is set. + boot_disk_type: Type of the boot disk (default is "pd-ssd"). Valid values: "pd-ssd" (Persistent Disk Solid State Drive) or "pd-standard" (Persistent Disk Hard Disk Drive). boot_disk_type is set as a static value and cannot be changed as a pipeline parameter. + boot_disk_size_gb: Size in GB of the boot disk (default is 100GB). `boot_disk_size_gb` is set as a static value and cannot be changed as a pipeline parameter. + timeout: The maximum job running time. The default is 7 days. A duration in seconds with up to nine fractional digits, terminated by 's', for example: "3.5s". + restart_job_on_worker_restart: Restarts the entire CustomJob if a worker gets restarted. This feature can be used by distributed training jobs that are not resilient to workers leaving and joining a job. + service_account: Sets the default service account for workload run-as account. The [service account](https://cloud.google.com/vertex-ai/docs/pipelines/configure-project#service-account) running the pipeline submitting jobs must have act-as permission on this run-as account. If unspecified, the Vertex AI Custom Code [Service Agent](https://cloud.google.com/vertex-ai/docs/general/access-control#service-agents) for the CustomJob's project. + network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Format is of the form `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `12345`, and `{network}` is a network name. Private services access must already be configured for the network. If left unspecified, the job is not peered with any network. + encryption_spec_key_name: Customer-managed encryption key options for the CustomJob. If this is set, then all resources created by the CustomJob will be encrypted with the provided encryption key. + tensorboard: The name of a Vertex AI TensorBoard resource to which this CustomJob will upload TensorBoard logs. + enable_web_access: Whether you want Vertex AI to enable [interactive shell access](https://cloud.google.com/vertex-ai/docs/training/monitor-debug-interactive-shell) to training containers. If `True`, you can access interactive shells at the URIs given by [CustomJob.web_access_uris][]. + reserved_ip_ranges: A list of names for the reserved IP ranges under the VPC network that can be used for this job. If set, we will deploy the job within the provided IP ranges. Otherwise, the job will be deployed to any IP ranges under the provided VPC network. + nfs_mounts: A list of [NfsMount](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/CustomJobSpec#NfsMount) resource specs in Json dict format. For more details about mounting NFS for CustomJob, see [Mount an NFS share for custom training](https://cloud.google.com/vertex-ai/docs/training/train-nfs-share). + base_output_directory: The Cloud Storage location to store the output of this CustomJob or HyperparameterTuningJob. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/GcsDestination). + labels: The labels with user-defined metadata to organize the CustomJob. See [more information](https://goo.gl/xmQnxf). + + Returns: + A KFP component with CustomJob specification applied. """ + # fmt: on # This function constructs a Custom Job component based on the input # component, by performing a 3-way merge of the inputs/outputs of the # input component, the Custom Job component and the arguments given to this @@ -330,13 +267,6 @@ def create_custom_training_job_op_from_component(*args, **kwargs) -> Callable: """Deprecated. Please use create_custom_training_job_from_component instead. - - Args: - *args: Positional arguments for create_custom_training_job_from_component. - **kwargs: Keyword arguments for create_custom_training_job_from_component. - - Returns: - A KFP component with CustomJob features applied. """ warnings.warn( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/__init__.py index d223ac39d0..110b890585 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/__init__.py @@ -11,10 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Create `Google Cloud Dataflow `_ jobs from within Vertex AI Pipelines.""" +# fmt: off +"""Create [Google Cloud Dataflow](https://cloud.google.com/dataflow) jobs from within Vertex AI Pipelines.""" +# fmt: on +from google_cloud_pipeline_components.v1.dataflow.flex_template.component import dataflow_flex_template as DataflowFlexTemplateJobOp from google_cloud_pipeline_components.v1.dataflow.python_job.component import dataflow_python as DataflowPythonJobOp __all__ = [ 'DataflowPythonJobOp', + 'DataflowFlexTemplateJobOp', ] diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/dataflow/flex_template/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/flex_template/__init__.py similarity index 100% rename from components/google-cloud/google_cloud_pipeline_components/preview/dataflow/flex_template/__init__.py rename to components/google-cloud/google_cloud_pipeline_components/v1/dataflow/flex_template/__init__.py diff --git a/components/google-cloud/google_cloud_pipeline_components/preview/dataflow/flex_template/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/flex_template/component.py similarity index 58% rename from components/google-cloud/google_cloud_pipeline_components/preview/dataflow/flex_template/component.py rename to components/google-cloud/google_cloud_pipeline_components/v1/dataflow/flex_template/component.py index 80b56b729f..239609c9e7 100644 --- a/components/google-cloud/google_cloud_pipeline_components/preview/dataflow/flex_template/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/flex_template/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component from kfp.dsl import ContainerSpec @@ -23,7 +24,6 @@ @container_component def dataflow_flex_template( - project: str, container_spec_gcs_path: str, gcp_resources: OutputPath(str), location: str = 'us-central1', @@ -56,111 +56,47 @@ def dataflow_flex_template( update: bool = False, transform_name_mappings: Dict[str, str] = {}, validate_only: bool = False, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a job with a Dataflow Flex Template. Args: - project: The ID of the Cloud Platform project that the job - belongs to. - location: The regional endpoint to which to direct the request. E.g., us-central1, - us-west1. Defaults to `us-central1` if not set. - job_name: The job name to use for the created job. For update job requests, the job - name should be the same as the existing running job. If none is specified, - a default name will be generated by the component. - container_spec_gcs_path: Cloud Storage path to a file with json serialized ContainerSpec as - content. - parameters: - The parameters for the flex template. Ex. {"my_template_param":"5"} - launch_options: - Launch options for this flex template job. This is a common set of options - across languages and templates. This should not be used to pass job - parameters. - num_workers: The initial number of Google Compute Engine instances for the job. If - empty or unspecified, the Dataflow service determines an appropriate - number of workers. - max_workers: The maximum number of Google Compute Engine instances to be made available - to your pipeline during execution, from 1 to 1000. If empty or - unspecified, the Dataflow service determines a default maximum number of - instances. For more details, see - https://cloud.google.com/dataflow/docs/horizontal-autoscaling. - service_account_email: The email address of the service account to run the job as. If - unspecified, the Dataflow service uses the project's Compute Engine - default service account. - temp_location: The Cloud Storage path to use for temporary files. Must be a valid Cloud - Storage URL, beginning with gs://. For more details, see - https://cloud.google.com/dataflow/docs/guides/setting-pipeline-options#setting_required_options. - machine_type: The machine type to use for the Dataflow job. Defaults to the value from - the template if not specified. + location: The regional endpoint to which to direct the request. E.g., us-central1, us-west1. Defaults to `us-central1` if not set. + job_name: The job name to use for the created job. For update job requests, the job name should be the same as the existing running job. If none is specified, a default name will be generated by the component. + container_spec_gcs_path: Cloud Storage path to a file with json serialized ContainerSpec as content. + parameters: The parameters for the flex template. Ex. {"my_template_param":"5"} + launch_options: Launch options for this flex template job. This is a common set of options across languages and templates. This should not be used to pass job parameters. + num_workers: The initial number of Google Compute Engine instances for the job. If empty or unspecified, the Dataflow service determines an appropriate number of workers. + max_workers: The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000. If empty or unspecified, the Dataflow service determines a default maximum number of instances. For more details, see https://cloud.google.com/dataflow/docs/horizontal-autoscaling. + service_account_email: The email address of the service account to run the job as. If unspecified, the Dataflow service uses the project's Compute Engine default service account. + temp_location: The Cloud Storage path to use for temporary files. Must be a valid Cloud Storage URL, beginning with gs://. For more details, see https://cloud.google.com/dataflow/docs/guides/setting-pipeline-options#setting_required_options. + machine_type: The machine type to use for the Dataflow job. Defaults to the value from the template if not specified. additional_experiments: Additional experiment flags for the job. - network: Network to which VMs will be assigned. If empty or unspecified, the - service will use the network "default". - subnetwork: Subnetwork to which VMs will be assigned, if desired. You can specify a - subnetwork using either a complete URL or an abbreviated path. - Expected to be of the form - "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK" - or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located - in a Shared VPC network, you must use the complete URL. - additional_user_labels: - Additional user labels to be specified for the job. Keys and values must - follow the restrictions specified in the labeling restrictions page - (https://cloud.google.com/compute/docs/labeling-resources#restrictions). - An object containing a list of "key": value pairs. - Example: { "name": "wrench", "mass": "1kg", "count": "3" }. - kms_key_name: Name for the Cloud KMS key for the job. Key format is - "projects/HOST_PROJECT_ID/locations/LOCATION/keyRings/KEYRING_ID/cryptoKeys/CRYPTO_KEY_ID" + network: Network to which VMs will be assigned. If empty or unspecified, the service will use the network "default". + subnetwork: Subnetwork to which VMs will be assigned, if desired. You can specify a subnetwork using either a complete URL or an abbreviated path. Expected to be of the form "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK" or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in a Shared VPC network, you must use the complete URL. + additional_user_labels: Additional user labels to be specified for the job. Keys and values must follow the restrictions specified in the labeling restrictions page (https://cloud.google.com/compute/docs/labeling-resources#restrictions). An object containing a list of "key": value pairs. Example: `{ "name": "wrench", "mass": "1kg", "count": "3" }`. + kms_key_name: Name for the Cloud KMS key for the job. Key format is "projects/HOST_PROJECT_ID/locations/LOCATION/keyRings/KEYRING_ID/cryptoKeys/CRYPTO_KEY_ID" ip_configuration: Configuration for VM IPs. - worker_region: The Compute Engine region - (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in - which worker processing should occur, e.g. "us-west1". Mutually exclusive - with worker_zone. If neither worker_region nor worker_zone is specified, - default to the control plane's region. - worker_zone: The Compute Engine zone - (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in - which worker processing should occur, e.g. "us-west1-a". Mutually - exclusive with workerRegion. If neither worker_region nor worker_zone is - specified, a zone in the control plane's region is chosen based on - available capacity. + worker_region: The Compute Engine region (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in which worker processing should occur, e.g. "us-west1". Mutually exclusive with worker_zone. If neither worker_region nor worker_zone is specified, default to the control plane's region. + worker_zone: The Compute Engine zone (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in which worker processing should occur, e.g. "us-west1-a". Mutually exclusive with workerRegion. If neither worker_region nor worker_zone is specified, a zone in the control plane's region is chosen based on available capacity. enable_streaming_engine: Whether to enable Streaming Engine for the job. - flexrs_goal: Set FlexRS goal for the job. For more details, see - https://cloud.google.com/dataflow/docs/guides/flexrs. - staging_location: The Cloud Storage path for staging local files. Must be a valid Cloud - Storage URL, beginning with gs://. For more details, see - https://cloud.google.com/dataflow/docs/guides/setting-pipeline-options#setting_required_options. - sdk_container_image: Docker registry location (e.g. Artifact Registry) of the container image - to use for the worker harness. Default is the container for the version of - the SDK. Note this field is only valid for portable Dataflow pipeline - jobs. - disk_size_gb: Worker disk size, in gigabytes. If empty or unspecified, the Dataflow - service determines an appropriate disk size. - autoscaling_algorithm: The algorithm to use for autoscaling. If empty or unspecified, the - Dataflow service sets a default value. For more details, see - https://cloud.google.com/dataflow/docs/reference/pipeline-options#resource_utilization. - dump_heap_on_oom: If true, when processing time is spent almost entirely on garbage - collection (GC), saves a heap dump before ending the thread or process. - If false, ends the thread or process without saving a heap dump. Does not - save a heap dump when the Java Virtual Machine (JVM) has an out of memory - error during processing. The location of the heap file is either echoed - back to the user, or the user is given the opportunity to download the - heap file. - save_heap_dumps_to_gcs_path: Cloud Storage bucket (directory) to upload heap dumps to. Enabling this - field implies that dump_heap_on_oom is set to true. - launcher_machine_type: The machine type to use for launching the Dataflow job. The default is - n1-standard-1. + flexrs_goal: Set FlexRS goal for the job. For more details, see https://cloud.google.com/dataflow/docs/guides/flexrs. + staging_location: The Cloud Storage path for staging local files. Must be a valid Cloud Storage URL, beginning with gs://. For more details, see https://cloud.google.com/dataflow/docs/guides/setting-pipeline-options#setting_required_options. + sdk_container_image: Docker registry location (e.g. Artifact Registry) of the container image to use for the worker harness. Default is the container for the version of the SDK. Note this field is only valid for portable Dataflow pipeline jobs. + disk_size_gb: Worker disk size, in gigabytes. If empty or unspecified, the Dataflow service determines an appropriate disk size. + autoscaling_algorithm: The algorithm to use for autoscaling. If empty or unspecified, the Dataflow service sets a default value. For more details, see https://cloud.google.com/dataflow/docs/reference/pipeline-options#resource_utilization. + dump_heap_on_oom: If true, when processing time is spent almost entirely on garbage collection (GC), saves a heap dump before ending the thread or process. If false, ends the thread or process without saving a heap dump. Does not save a heap dump when the Java Virtual Machine (JVM) has an out of memory error during processing. The location of the heap file is either echoed back to the user, or the user is given the opportunity to download the heap file. + save_heap_dumps_to_gcs_path: Cloud Storage bucket (directory) to upload heap dumps to. Enabling this field implies that dump_heap_on_oom is set to true. + launcher_machine_type: The machine type to use for launching the Dataflow job. The default is n1-standard-1. enable_launcher_vm_serial_port_logging: If true serial port logging will be enabled for the launcher VM. - update: Set this to true if you are sending a request to update a running - streaming job. When set, the job name should be the same as the running - job. - transform_name_mappings: - Use this to pass transformNameMappings for streaming update jobs. - Ex:{"oldTransformName":"newTransformName",...}'. For more details, see - https://cloud.google.com/dataflow/docs/guides/updating-a-pipeline#Mapping - validate_only: If true, the request is validated but not actually executed. Defaults to - false. + update: Set this to true if you are sending a request to update a running streaming job. When set, the job name should be the same as the running job. + transform_name_mappings: Use this to pass transformNameMappings for streaming update jobs. Example: `{"oldTransformName":"newTransformName",...}`. For more details, see https://cloud.google.com/dataflow/docs/guides/updating-a-pipeline#Mapping + validate_only: If true, the request is validated but not actually executed. Defaults to false. + project: The ID of the Cloud Platform project that the job belongs to. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/python_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/python_job/component.py index 48f8e5aa49..5f965d8814 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/python_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataflow/python_job/component.py @@ -14,6 +14,7 @@ from typing import List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from kfp.dsl import container_component from kfp.dsl import ContainerSpec from kfp.dsl import OutputPath @@ -21,32 +22,28 @@ @container_component def dataflow_python( - project: str, python_module_path: str, temp_location: str, gcp_resources: OutputPath(str), location: str = 'us-central1', requirements_file_path: str = '', args: List[str] = [], + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Launch a self-executing Beam Python file on Google Cloud using the Dataflow Runner. Args: - project: Project to create the Dataflow job. - location: Location of the Dataflow job. If not set, defaults to - ``'us-central1'``. + location: Location of the Dataflow job. If not set, defaults to `'us-central1'`. python_module_path: The GCS path to the Python file to run. - temp_location: A GCS path for Dataflow to stage temporary job - files created during the execution of the pipeline. + temp_location: A GCS path for Dataflow to stage temporary job files created during the execution of the pipeline. requirements_file_path: The GCS path to the pip requirements file. - args: The list of args to pass to the Python file. Can include additional - parameters for the Dataflow Runner. + args: The list of args to pass to the Python file. Can include additional parameters for the Dataflow Runner. + project: Project to create the Dataflow job. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/__init__.py index 89d3f2c3f9..c739fb403a 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Create `Google Cloud Dataproc `_ jobs from within Vertex AI Pipelines.""" +# fmt: off +"""Create [Google Cloud Dataproc](https://cloud.google.com/dataproc) jobs from within Vertex AI Pipelines.""" +# fmt: on from google_cloud_pipeline_components.v1.dataproc.create_pyspark_batch.component import dataproc_create_pyspark_batch as DataprocPySparkBatchOp from google_cloud_pipeline_components.v1.dataproc.create_spark_batch.component import dataproc_create_spark_batch as DataprocSparkBatchOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_pyspark_batch/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_pyspark_batch/component.py index bb210c49a7..73efc04c23 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_pyspark_batch/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_pyspark_batch/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component from kfp.dsl import ContainerSpec @@ -23,7 +24,6 @@ @container_component def dataproc_create_pyspark_batch( - project: str, main_python_file_uri: str, gcp_resources: OutputPath(str), location: str = 'us-central1', @@ -44,60 +44,35 @@ def dataproc_create_pyspark_batch( file_uris: List[str] = [], archive_uris: List[str] = [], args: List[str] = [], + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Create a Dataproc PySpark batch workload and wait for it to finish. Args: - project: Project to run the Dataproc batch workload. - location: Location of the Dataproc batch workload. If - not set, defaults to ``"us-central1"``. - batch_id: The ID to use for the batch, which will become - the final component of the batch's resource name. If none is - specified, a default name will be generated by the component. This - value must be 4-63 characters. Valid characters are ``/[a-z][0-9]-/``. - labels: The labels to associate with this batch. Label - keys must contain 1 to 63 characters, and must conform to RFC 1035. - Label values may be empty, but, if present, must contain 1 to 63 - characters, and must conform to RFC 1035. No more than 32 labels can - be associated with a batch. An object containing a list of ``"key": - value`` pairs. - Example: ``{ "name": "wrench", "mass": "1.3kg", "count": "3" }``. - container_image: Optional custom container image for the - job runtime environment. If not specified, a default container image - will be used. + location: Location of the Dataproc batch workload. If not set, defaults to `"us-central1"`. + batch_id: The ID to use for the batch, which will become the final component of the batch's resource name. If none is specified, a default name will be generated by the component. This value must be 4-63 characters. Valid characters are `/[a-z][0-9]-/`. + labels: The labels to associate with this batch. Label keys must contain 1 to 63 characters, and must conform to RFC 1035. Label values may be empty, but, if present, must contain 1 to 63 characters, and must conform to RFC 1035. No more than 32 labels can be associated with a batch. An object containing a list of `"key": value` pairs. Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`. + container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used. runtime_config_version: Version of the batch runtime. runtime_config_properties: Runtime configuration for the workload. service_account: Service account that is used to execute the workload. - network_tags: Tags used for network traffic - control. + network_tags: Tags used for network traffic control. kms_key: The Cloud KMS key to use for encryption. network_uri: Network URI to connect workload to. subnetwork_uri: Subnetwork URI to connect workload to. - metastore_service: Resource name of an existing Dataproc - Metastore service. - spark_history_dataproc_cluster: The Spark History Server - configuration for the workload. - main_python_file_uri: The HCFS URI of the main Python - file to use as the Spark driver. Must be a ``.py`` file. - python_file_uris: HCFS file URIs of Python files to - pass to the PySpark framework. Supported file types: ``.py``, ``.egg``, - and ``.zip``. - jar_file_uris: HCFS URIs of jar files to add to the - classpath of the Spark driver and tasks. - file_uris: HCFS URIs of files to be placed in the - working directory of each executor. - archive_uris: HCFS URIs of archives to be extracted - into the working directory of each executor. Supported file types: - ``.jar``, ``.tar``, ``.tar.gz``, ``.tgz``, and ``.zip``. - args: The arguments to pass to the driver. Do not - include arguments that can be set as batch properties, such as - ``--conf``, since a collision can occur that causes an incorrect batch - submission. + metastore_service: Resource name of an existing Dataproc Metastore service. + spark_history_dataproc_cluster: The Spark History Server configuration for the workload. + main_python_file_uri: The HCFS URI of the main Python file to use as the Spark driver. Must be a `.py` file. + python_file_uris: HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: `.py`, `.egg`, and `.zip`. + jar_file_uris: HCFS URIs of jar files to add to the classpath of the Spark driver and tasks. + file_uris: HCFS URIs of files to be placed in the working directory of each executor. + archive_uris: HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: `.jar`, `.tar`, `.tar.gz`, `.tgz`, and `.zip`. + args: The arguments to pass to the driver. Do not include arguments that can be set as batch properties, such as `--conf`, since a collision can occur that causes an incorrect batch submission. + project: Project to run the Dataproc batch workload. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_batch/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_batch/component.py index 8bb14a8d7d..7d705247d5 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_batch/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_batch/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component from kfp.dsl import ContainerSpec @@ -23,7 +24,6 @@ @container_component def dataproc_create_spark_batch( - project: str, gcp_resources: OutputPath(str), location: str = 'us-central1', batch_id: str = '', @@ -44,60 +44,35 @@ def dataproc_create_spark_batch( file_uris: List[str] = [], archive_uris: List[str] = [], args: List[str] = [], + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Create a Dataproc Spark batch workload and wait for it to finish. Args: - project: Project to run the Dataproc batch workload. - location: Location of the Dataproc batch workload. If - not set, defaults to ``"us-central1"``. - batch_id: The ID to use for the batch, which will become - the final component of the batch's resource name. If none is - specified, a default name will be generated by the component. This - value must be 4-63 characters. Valid characters are ``/[a-z][0-9]-/``. - labels: The labels to associate with this batch. Label - keys must contain 1 to 63 characters, and must conform to RFC 1035. - Label values may be empty, but, if present, must contain 1 to 63 - characters, and must conform to RFC 1035. No more than 32 labels can - be associated with a batch. An object containing a list of ``"key": - value`` pairs. - Example: ``{ "name": "wrench", "mass": "1.3kg", "count": "3" }``. - container_image: Optional custom container image for the - job runtime environment. If not specified, a default container image - will be used. + location: Location of the Dataproc batch workload. If not set, defaults to `"us-central1"`. + batch_id: The ID to use for the batch, which will become the final component of the batch's resource name. If none is specified, a default name will be generated by the component. This value must be 4-63 characters. Valid characters are `/[a-z][0-9]-/`. + labels: The labels to associate with this batch. Label keys must contain 1 to 63 characters, and must conform to RFC 1035. Label values may be empty, but, if present, must contain 1 to 63 characters, and must conform to RFC 1035. No more than 32 labels can be associated with a batch. An object containing a list of `"key": value` pairs. Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`. + container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used. runtime_config_version: Version of the batch runtime. runtime_config_properties: Runtime configuration for the workload. service_account: Service account that is used to execute the workload. - network_tags: Tags used for network traffic - control. + network_tags: Tags used for network traffic control. kms_key: The Cloud KMS key to use for encryption. network_uri: Network URI to connect workload to. subnetwork_uri: Subnetwork URI to connect workload to. - metastore_service: Resource name of an existing Dataproc - Metastore service. - spark_history_dataproc_cluster: The Spark History Server - configuration for the workload. - main_jar_file_uri: The HCFS URI of the jar file that - contains the main class. - main_class: The name of the driver main class. The jar - file that contains the class must be in the classpath or specified in - jar_file_uris. - jar_file_uris: HCFS URIs of jar files to add to the classpath of the Spark - driver and tasks. - file_uris: HCFS URIs of files to be placed in the working directory of - each executor. - archive_uris: HCFS URIs of archives to be extracted into the working - directory of each executor. Supported file types: - ``.jar``, ``.tar``, ``.tar.gz``, ``.tgz``, and ``.zip``. - args: The arguments to pass to the driver. Do not - include arguments that can be set as batch properties, such as - ``--conf``, since a collision can occur that causes an incorrect batch - submission. + metastore_service: Resource name of an existing Dataproc Metastore service. + spark_history_dataproc_cluster: The Spark History Server configuration for the workload. + main_jar_file_uri: The HCFS URI of the jar file that contains the main class. + main_class: The name of the driver main class. The jar file that contains the class must be in the classpath or specified in jar_file_uris. + jar_file_uris: HCFS URIs of jar files to add to the classpath of the Spark driver and tasks. + file_uris: HCFS URIs of files to be placed in the working directory of each executor. + archive_uris: HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: `.jar`, `.tar`, `.tar.gz`, `.tgz`, and `.zip`. + args: The arguments to pass to the driver. Do not include arguments that can be set as batch properties, such as `--conf`, since a collision can occur that causes an incorrect batch submission. + project: Project to run the Dataproc batch workload. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_r_batch/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_r_batch/component.py index a599780af1..8ed58fd66d 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_r_batch/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_r_batch/component.py @@ -15,6 +15,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component from kfp.dsl import ContainerSpec @@ -23,7 +24,6 @@ @container_component def dataproc_create_spark_r_batch( - project: str, gcp_resources: OutputPath(str), location: str = 'us-central1', batch_id: str = '', @@ -42,28 +42,16 @@ def dataproc_create_spark_r_batch( file_uris: List[str] = [], archive_uris: List[str] = [], args: List[str] = [], + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Create a Dataproc SparkR batch workload and wait for it to finish. Args: - project: Project to run the Dataproc batch workload. - location: Location of the Dataproc batch workload. If not set, defaults to - ``"us-central1"``. - batch_id: The ID to use for the batch, which will become - the final component of the batch's resource name. If none is - specified, a default name will be generated by the component. This - value must be 4-63 characters. Valid characters are ``/[a-z][0-9]-/``. - labels: The labels to associate with this batch. Label - keys must contain 1 to 63 characters, and must conform to RFC 1035. - Label values may be empty, but, if present, must contain 1 to 63 - characters, and must conform to RFC 1035. No more than 32 labels can - be associated with a batch. An object containing a list of ``"key": - value`` pairs. - Example: ``{ "name": "wrench", "mass": "1.3kg", "count": "3" }``. - container_image: Optional custom container image for the - job runtime environment. If not specified, a default container image - will be used. + location: Location of the Dataproc batch workload. If not set, defaults to `"us-central1"`. + batch_id: The ID to use for the batch, which will become the final component of the batch's resource name. If none is specified, a default name will be generated by the component. This value must be 4-63 characters. Valid characters are `/[a-z][0-9]-/`. + labels: The labels to associate with this batch. Label keys must contain 1 to 63 characters, and must conform to RFC 1035. Label values may be empty, but, if present, must contain 1 to 63 characters, and must conform to RFC 1035. No more than 32 labels can be associated with a batch. An object containing a list of `"key": value` pairs. Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`. + container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used. runtime_config_version: Version of the batch runtime. runtime_config_properties: Runtime configuration for the workload. service_account: Service account that is used to execute the workload. @@ -71,25 +59,16 @@ def dataproc_create_spark_r_batch( kms_key: The Cloud KMS key to use for encryption. network_uri: Network URI to connect workload to. subnetwork_uri: Subnetwork URI to connect workload to. - metastore_service: Resource name of an existing Dataproc Metastore - service. - spark_history_dataproc_cluster: The Spark History Server configuration for - the workload. - main_r_file_uri: The HCFS URI of the main R file to use as the driver. - Must be a ``.R`` or ``.r`` file. - file_uris: HCFS URIs of files to be placed in the working directory of - each executor. - archive_uris: HCFS URIs of archives to be extracted into the working - directory of each executor. Supported file types: - ``.jar``, ``.tar``, ``.tar.gz``, ``.tgz``, and ``.zip``. - args: The arguments to pass to the driver. Do not - include arguments that can be set as batch properties, such as - ``--conf``, since a collision can occur that causes an incorrect batch - submission. + metastore_service: Resource name of an existing Dataproc Metastore service. + spark_history_dataproc_cluster: The Spark History Server configuration for the workload. + main_r_file_uri: The HCFS URI of the main R file to use as the driver. Must be a `.R` or `.r` file. + file_uris: HCFS URIs of files to be placed in the working directory of each executor. + archive_uris: HCFS URIs of archives to be extracted into the working directory of each executor. Supported file types: `.jar`, `.tar`, `.tar.gz`, `.tgz`, and `.zip`. + args: The arguments to pass to the driver. Do not include arguments that can be set as batch properties, such as `--conf`, since a collision can occur that causes an incorrect batch submission. + project: Project to run the Dataproc batch workload. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_sql_batch/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_sql_batch/component.py index 51e31fa9a5..76611cd64e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_sql_batch/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataproc/create_spark_sql_batch/component.py @@ -14,6 +14,7 @@ from typing import Dict, List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component from kfp.dsl import ContainerSpec @@ -22,7 +23,6 @@ @container_component def dataproc_create_spark_sql_batch( - project: str, gcp_resources: OutputPath(str), location: str = 'us-central1', batch_id: str = '', @@ -40,28 +40,16 @@ def dataproc_create_spark_sql_batch( query_file_uri: str = '', query_variables: Dict[str, str] = {}, jar_file_uris: List[str] = [], + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off """Create a Dataproc Spark SQL batch workload and wait for it to finish. Args: - project: Project to run the Dataproc batch workload. - location: Location of the Dataproc batch workload. If - not set, defaults to ``"us-central1"``. - batch_id: The ID to use for the batch, which will become - the final component of the batch's resource name. If none is - specified, a default name will be generated by the component. This - value must be 4-63 characters. Valid characters are /[a-z][0-9]-/. - labels: The labels to associate with this batch. Label - keys must contain 1 to 63 characters, and must conform to RFC 1035. - Label values may be empty, but, if present, must contain 1 to 63 - characters, and must conform to RFC 1035. No more than 32 labels can - be associated with a batch. An object containing a list of ``"key": - value`` pairs. - Example: ``{ "name": "wrench", "mass": "1.3kg", "count": "3" }``. - container_image: Optional custom container image for the - job runtime environment. If not specified, a default container image - will be used. + location: Location of the Dataproc batch workload. If not set, defaults to `"us-central1"`. + batch_id: The ID to use for the batch, which will become the final component of the batch's resource name. If none is specified, a default name will be generated by the component. This value must be 4-63 characters. Valid characters are /[a-z][0-9]-/. + labels: The labels to associate with this batch. Label keys must contain 1 to 63 characters, and must conform to RFC 1035. Label values may be empty, but, if present, must contain 1 to 63 characters, and must conform to RFC 1035. No more than 32 labels can be associated with a batch. An object containing a list of `"key": value` pairs. Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`. + container_image: Optional custom container image for the job runtime environment. If not specified, a default container image will be used. runtime_config_version: Version of the batch runtime. runtime_config_properties: Runtime configuration for the workload. service_account: Service account that is used to execute the workload. @@ -69,22 +57,15 @@ def dataproc_create_spark_sql_batch( kms_key: The Cloud KMS key to use for encryption. network_uri: Network URI to connect workload to. subnetwork_uri: Subnetwork URI to connect workload to. - metastore_service: Resource name of an existing Dataproc Metastore - service. - spark_history_dataproc_cluster: The Spark History Server configuration for - the workload. - query_file_uri: The HCFS URI of the script that contains Spark SQL queries - to execute. - query_variables: Mapping of query variable names to values (equivalent to - the Spark SQL command: ``SET name="value";``). An object containing a - list of ``"key": value`` pairs. - Example: ``{ "name": "wrench", "mass": "1.3kg", "count": "3" }``. - jar_file_uris: HCFS URIs of jar files to be added to the Spark - ``CLASSPATH``. + metastore_service: Resource name of an existing Dataproc Metastore service. + spark_history_dataproc_cluster: The Spark History Server configuration for the workload. + query_file_uri: The HCFS URI of the script that contains Spark SQL queries to execute. + query_variables: Mapping of query variable names to values (equivalent to the Spark SQL command: `SET name="value";`). An object containing a list of `"key": value` pairs. Example: `{ "name": "wrench", "mass": "1.3kg", "count": "3" }`. + jar_file_uris: HCFS URIs of jar files to be added to the Spark `CLASSPATH`. + project: Project to run the Dataproc batch workload. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + gcp_resources: Serialized gcp_resources proto tracking the Dataproc batch workload. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/__init__.py index 641ed5ec04..5bb4c1a7c0 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Manage datasets via `Vertex AI Datasets `_.""" +# fmt: off +"""Manage datasets via [Vertex AI Datasets](https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets).""" +# fmt: on from google_cloud_pipeline_components.v1.dataset.create_image_dataset.component import image_dataset_create as ImageDatasetCreateOp from google_cloud_pipeline_components.v1.dataset.create_tabular_dataset.component import tabular_dataset_create as TabularDatasetCreateOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_image_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_image_dataset/component.py index 345a7514e0..b1bef2a3fa 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_image_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_image_dataset/component.py @@ -15,6 +15,7 @@ from typing import Dict, Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Output @@ -22,7 +23,6 @@ @dsl.container_component def image_dataset_create( - project: str, display_name: str, dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', @@ -31,57 +31,20 @@ def image_dataset_create( import_schema_uri: Optional[str] = None, labels: Optional[Dict[str, str]] = {}, encryption_spec_key_name: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Creates a new image `Dataset `_ and optionally imports data into Dataset when - ``source`` and ``import_schema_uri`` are passed. + """Creates a new image [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) and optionally imports data into Dataset when `source` and `import_schema_uri` are passed. Args: - display_name: The user-defined name of the Dataset. - The name can be up to 128 characters long and can be consist - of any UTF-8 characters. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, ``"gs://bucket/file.csv"`` or ``["gs://bucket/file1.csv", "gs://bucket/file2.csv"]``. - import_schema_uri: Points to a YAML file stored on Google Cloud - Storage describing the import format. Validation will be - done against the schema. The schema is defined as an - `OpenAPI 3.0.2 Schema Object `_. - data_item_labels: Labels that will be applied to newly imported DataItems. If - an identical DataItem as one being imported already exists - in the Dataset, then these labels will be appended to these - of the already existing one, and if labels with identical - key is imported before, the old label value will be - overwritten. If two DataItems are identical in the same - import data operation, the labels will be combined and if - key collision happens in this case, one of the values will - be picked randomly. Two DataItems are considered identical - if their content bytes are identical (e.g. image bytes or - pdf bytes). These labels will be overridden by Annotation - labels specified inside index file refenced by - ``import_schema_uri``, e.g. jsonl file. - project: Project to retrieve Dataset from. + display_name: The user-defined name of the Dataset. The name can be up to 128 characters long and can be consist of any UTF-8 characters. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, `"gs://bucket/file.csv"` or `["gs://bucket/file1.csv", "gs://bucket/file2.csv"]`. + import_schema_uri: Points to a YAML file stored on Google Cloud Storage describing the import format. Validation will be done against the schema. The schema is defined as an [OpenAPI 3.0.2 Schema Object](https://tinyurl.com/y538mdwt). + data_item_labels: Labels that will be applied to newly imported DataItems. If an identical DataItem as one being imported already exists in the Dataset, then these labels will be appended to these of the already existing one, and if labels with identical key is imported before, the old label value will be overwritten. If two DataItems are identical in the same import data operation, the labels will be combined and if key collision happens in this case, one of the values will be picked randomly. Two DataItems are considered identical if their content bytes are identical (e.g. image bytes or pdf bytes). These labels will be overridden by Annotation labels specified inside index file refenced by `import_schema_uri`, e.g. jsonl file. location: Optional location to retrieve Dataset from. - labels: Labels with user-defined metadata to organize your Tensorboards. - Label keys and values can be no longer than 64 characters - (Unicode codepoints), can only contain lowercase letters, numeric - characters, underscores and dashes. International characters are allowed. - No more than 64 user labels can be associated with one Tensorboard - (System labels are excluded). - See https://goo.gl/xmQnxf for more information and examples of labels. - System reserved label keys are prefixed with "aiplatform.googleapis.com/" - and are immutable. - encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the Dataset. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this Dataset and all sub-resources of this Dataset will be secured by this key. - Overrides ``encryption_spec_key_name`` set in ``aiplatform.init``. + labels: Labels with user-defined metadata to organize your Tensorboards. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Tensorboard (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. + encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the Dataset. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this Dataset and all sub-resources of this Dataset will be secured by this key. Overrides `encryption_spec_key_name` set in `aiplatform.init`. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: dataset: Instantiated representation of the managed image Dataset resource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_tabular_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_tabular_dataset/component.py index 3f19637740..4dcc1e7513 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_tabular_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_tabular_dataset/component.py @@ -16,6 +16,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Output @@ -23,7 +24,6 @@ @dsl.container_component def tabular_dataset_create( - project: str, display_name: str, dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', @@ -31,40 +31,20 @@ def tabular_dataset_create( bq_source: Optional[str] = None, labels: Optional[dict] = {}, encryption_spec_key_name: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Creates a new tabular `Dataset `_. + """Creates a new tabular [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets). Args: - display_name: The user-defined name of the Dataset. - The name can be up to 128 characters long and can be consist - of any UTF-8 characters. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, ``"gs://bucket/file.csv"`` or ``["gs://bucket/file1.csv", "gs://bucket/file2.csv"]``. + display_name: The user-defined name of the Dataset. The name can be up to 128 characters long and can be consist of any UTF-8 characters. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, `"gs://bucket/file.csv"` or `["gs://bucket/file1.csv", "gs://bucket/file2.csv"]`. bq_source: BigQuery URI to the input table. For example, "bq://project.dataset.table_name". - project: Project to retrieve Dataset from. location: Optional location to retrieve Dataset from. - labels: Labels with user-defined metadata to organize your Tensorboards. - Label keys and values can be no longer than 64 characters - (Unicode codepoints), can only contain lowercase letters, numeric - characters, underscores and dashes. International characters are allowed. - No more than 64 user labels can be associated with one Tensorboard - (System labels are excluded). - See https://goo.gl/xmQnxf for more information and examples of labels. - System reserved label keys are prefixed with "aiplatform.googleapis.com/" - and are immutable. - encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the Dataset. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this Dataset and all sub-resources of this Dataset will be secured by this key. - Overrides ``encryption_spec_key_name`` set in ``aiplatform.init``. + labels: Labels with user-defined metadata to organize your Tensorboards. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Tensorboard (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. + encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the Dataset. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this Dataset and all sub-resources of this Dataset will be secured by this key. Overrides `encryption_spec_key_name` set in `aiplatform.init`. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. + Returns: dataset: Instantiated representation of the managed tabular Dataset resource. """ diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_text_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_text_dataset/component.py index 93395aca13..4819d39b9e 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_text_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_text_dataset/component.py @@ -16,6 +16,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Output @@ -23,7 +24,6 @@ @dsl.container_component def text_dataset_create( - project: str, display_name: str, dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', @@ -32,57 +32,22 @@ def text_dataset_create( import_schema_uri: Optional[str] = None, labels: Optional[dict] = {}, encryption_spec_key_name: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Creates a new text `Dataset `_ and optionally imports data into Dataset when - ``source`` and ``import_schema_uri`` are passed. + """Creates a new text [Dataset](https://cloud.google.com/vertex- + ai/docs/reference/rest/v1/projects.locations.datasets) and optionally imports + data into Dataset when `source` and `import_schema_uri` are passed. Args: - display_name: The user-defined name of the Dataset. - The name can be up to 128 characters long and can be consist - of any UTF-8 characters. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, ``"gs://bucket/file.csv"`` or ``["gs://bucket/file1.csv", "gs://bucket/file2.csv"]``. - import_schema_uri: Points to a YAML file stored on Google Cloud - Storage describing the import format. Validation will be - done against the schema. The schema is defined as an - `OpenAPI 3.0.2 Schema Object `_. - data_item_labels: Labels that will be applied to newly imported DataItems. If - an identical DataItem as one being imported already exists - in the Dataset, then these labels will be appended to these - of the already existing one, and if labels with identical - key is imported before, the old label value will be - overwritten. If two DataItems are identical in the same - import data operation, the labels will be combined and if - key collision happens in this case, one of the values will - be picked randomly. Two DataItems are considered identical - if their content bytes are identical (e.g. image bytes or - pdf bytes). These labels will be overridden by Annotation - labels specified inside index file refenced by - ``import_schema_uri``, e.g. jsonl file. - project: Project to retrieve Dataset from. + display_name: The user-defined name of the Dataset. The name can be up to 128 characters long and can be consist of any UTF-8 characters. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, `"gs://bucket/file.csv"` or `["gs://bucket/file1.csv", "gs://bucket/file2.csv"]`. + import_schema_uri: Points to a YAML file stored on Google Cloud Storage describing the import format. Validation will be done against the schema. The schema is defined as an [OpenAPI 3.0.2 Schema Object](https://tinyurl.com/y538mdwt). + data_item_labels: Labels that will be applied to newly imported DataItems. If an identical DataItem as one being imported already exists in the Dataset, then these labels will be appended to these of the already existing one, and if labels with identical key is imported before, the old label value will be overwritten. If two DataItems are identical in the same import data operation, the labels will be combined and if key collision happens in this case, one of the values will be picked randomly. Two DataItems are considered identical if their content bytes are identical (e.g. image bytes or pdf bytes). These labels will be overridden by Annotation labels specified inside index file refenced by `import_schema_uri`, e.g. jsonl file. location: Optional location to retrieve Dataset from. - labels: Labels with user-defined metadata to organize your Tensorboards. - Label keys and values can be no longer than 64 characters - (Unicode codepoints), can only contain lowercase letters, numeric - characters, underscores and dashes. International characters are allowed. - No more than 64 user labels can be associated with one Tensorboard - (System labels are excluded). - See https://goo.gl/xmQnxf for more information and examples of labels. - System reserved label keys are prefixed with "aiplatform.googleapis.com/" - and are immutable. - encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the Dataset. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this Dataset and all sub-resources of this Dataset will be secured by this key. - Overrides ``encryption_spec_key_name`` set in ``aiplatform.init``. + labels: Labels with user-defined metadata to organize your Tensorboards. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Tensorboard (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. + encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the Dataset. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this Dataset and all sub-resources of this Dataset will be secured by this key. Overrides `encryption_spec_key_name` set in `aiplatform.init`. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: dataset: Instantiated representation of the managed text Datasetresource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_time_series_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_time_series_dataset/component.py index 93bc3ea3f9..1c8fb70d97 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_time_series_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_time_series_dataset/component.py @@ -16,6 +16,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Output @@ -23,7 +24,6 @@ @dsl.container_component def time_series_dataset_create( - project: str, display_name: str, dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', @@ -31,40 +31,19 @@ def time_series_dataset_create( bq_source: Optional[str] = None, labels: Optional[dict] = {}, encryption_spec_key_name: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Creates a new time series `Dataset `_. + """Creates a new time series [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets). Args: - display_name: The user-defined name of the Dataset. - The name can be up to 128 characters long and can be consist - of any UTF-8 characters. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, ``"gs://bucket/file.csv"`` or ``["gs://bucket/file1.csv", "gs://bucket/file2.csv"]``. + display_name: The user-defined name of the Dataset. The name can be up to 128 characters long and can be consist of any UTF-8 characters. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, `"gs://bucket/file.csv"` or `["gs://bucket/file1.csv", "gs://bucket/file2.csv"]`. bq_source: BigQuery URI to the input table. For example, bq://project.dataset.table_name". - project: Project to retrieve Dataset from. location: Optional location to retrieve Dataset from. - labels: Labels with user-defined metadata to organize your Tensorboards. - Label keys and values can be no longer than 64 characters - (Unicode codepoints), can only contain lowercase letters, numeric - characters, underscores and dashes. International characters are allowed. - No more than 64 user labels can be associated with one Tensorboard - (System labels are excluded). - See https://goo.gl/xmQnxf for more information and examples of labels. - System reserved label keys are prefixed with "aiplatform.googleapis.com/" - and are immutable. - encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the dataset. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this Dataset and all sub-resources of this Dataset will be secured by this key. - Overrides ``encryption_spec_key_name`` set in ``aiplatform.init``. + labels: Labels with user-defined metadata to organize your Tensorboards. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Tensorboard (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. + encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the dataset. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this Dataset and all sub-resources of this Dataset will be secured by this key. Overrides `encryption_spec_key_name` set in `aiplatform.init`. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: dataset: Instantiated representation of the managed time series Datasetresource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_video_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_video_dataset/component.py index e80bd89e9d..4789d9aa8d 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_video_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/create_video_dataset/component.py @@ -16,6 +16,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Output @@ -23,7 +24,6 @@ @dsl.container_component def video_dataset_create( - project: str, display_name: str, dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', @@ -32,58 +32,20 @@ def video_dataset_create( import_schema_uri: Optional[str] = None, labels: Optional[dict] = {}, encryption_spec_key_name: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Creates a new video `Dataset `_ and optionally imports data into Dataset when - ``source`` and ``import_schema_uri`` are passed. + """Creates a new video [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) and optionally imports data into Dataset when `source` and `import_schema_uri` are passed. Args: - display_name: The user-defined name of the Dataset. - The name can be up to 128 characters long and can be consist - of any UTF-8 characters. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, ``"gs://bucket/file.csv"`` or ``["gs://bucket/file1.csv", "gs://bucket/file2.csv"]``. - import_schema_uri: Points to a YAML file stored on Google Cloud - Storage describing the import format. Validation will be - done against the schema. The schema is defined as an - `OpenAPI 3.0.2 Schema - Object `_. - data_item_labels: Labels that will be applied to newly imported DataItems. If - an identical DataItem as one being imported already exists - in the Dataset, then these labels will be appended to these - of the already existing one, and if labels with identical - key is imported before, the old label value will be - overwritten. If two DataItems are identical in the same - import data operation, the labels will be combined and if - key collision happens in this case, one of the values will - be picked randomly. Two DataItems are considered identical - if their content bytes are identical (e.g. image bytes or - pdf bytes). These labels will be overridden by Annotation - labels specified inside index file refenced by - ``import_schema_uri``, - project: Project to retrieve Dataset from. + display_name: The user-defined name of the Dataset. The name can be up to 128 characters long and can be consist of any UTF-8 characters. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, `"gs://bucket/file.csv"` or `["gs://bucket/file1.csv", "gs://bucket/file2.csv"]`. + import_schema_uri: Points to a YAML file stored on Google Cloud Storage describing the import format. Validation will be done against the schema. The schema is defined as an [OpenAPI 3.0.2 Schema Object](https://tinyurl.com/y538mdwt). + data_item_labels: Labels that will be applied to newly imported DataItems. If an identical DataItem as one being imported already exists in the Dataset, then these labels will be appended to these of the already existing one, and if labels with identical key is imported before, the old label value will be overwritten. If two DataItems are identical in the same import data operation, the labels will be combined and if key collision happens in this case, one of the values will be picked randomly. Two DataItems are considered identical if their content bytes are identical (e.g. image bytes or pdf bytes). These labels will be overridden by Annotation labels specified inside index file refenced by `import_schema_uri`, location: Optional location to retrieve Dataset from. - labels: Labels with user-defined metadata to organize your Tensorboards. - Label keys and values can be no longer than 64 characters - (Unicode codepoints), can only contain lowercase letters, numeric - characters, underscores and dashes. International characters are allowed. - No more than 64 user labels can be associated with one Tensorboard - (System labels are excluded). - See https://goo.gl/xmQnxf for more information and examples of labels. - System reserved label keys are prefixed with "aiplatform.googleapis.com/" - and are immutable. - encryption_spec_key_name: The Cloud KMS resource identifier of the customer - managed encryption key used to protect the Dataset. Has the - form: - ``projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute - resource is created. - If set, this Dataset and all sub-resources of this Dataset will be secured by this key. - Overrides ``encryption_spec_key_name`` set in ``aiplatform.init``. + labels: Labels with user-defined metadata to organize your Tensorboards. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Tensorboard (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. + encryption_spec_key_name: The Cloud KMS resource identifier of the customer managed encryption key used to protect the Dataset. Has the form: `projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this Dataset and all sub-resources of this Dataset will be secured by this key. Overrides `encryption_spec_key_name` set in `aiplatform.init`. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: dataset: Instantiated representation of the managed video Datasetresource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_image_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_image_dataset/component.py index d53d51ecc6..f948067f64 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_image_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_image_dataset/component.py @@ -15,6 +15,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Input @@ -23,30 +24,19 @@ @dsl.container_component def image_dataset_export( - project: str, dataset: Input[VertexDataset], output_dir: str, exported_dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Exports `Dataset `_ to a GCS output directory. + """Exports [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) to a GCS output directory. Args: - output_dir: The Google Cloud Storage location where the output is to - be written to. In the given directory a new directory will be - created with name: - ``export-data--`` - where timestamp is in YYYYMMDDHHMMSS format. All export - output will be written into that directory. Inside that - directory, annotations with the same schema will be grouped - into sub directories which are named with the corresponding - annotations' schema title. Inside these sub directories, a - schema.yaml will be created to describe the output format. - If the uri doesn't end with '/', a '/' will be automatically - appended. The directory is created if it doesn't exist. - project: Project to retrieve Dataset from. + output_dir: The Google Cloud Storage location where the output is to be written to. In the given directory a new directory will be created with name: `export-data--` where timestamp is in YYYYMMDDHHMMSS format. All export output will be written into that directory. Inside that directory, annotations with the same schema will be grouped into sub directories which are named with the corresponding annotations' schema title. Inside these sub directories, a schema.yaml will be created to describe the output format. If the uri doesn't end with '/', a '/' will be automatically appended. The directory is created if it doesn't exist. location: Optional location to retrieve Dataset from. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: exported_dataset: All of the files that are exported in this export operation. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_tabular_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_tabular_dataset/component.py index 140a524743..aa33a38150 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_tabular_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_tabular_dataset/component.py @@ -15,6 +15,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Input @@ -23,30 +24,19 @@ @dsl.container_component def tabular_dataset_export( - project: str, dataset: Input[VertexDataset], output_dir: str, exported_dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Exports `Dataset `_ to a GCS output directory. + """Exports [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) to a GCS output directory. Args: - output_dir: The Google Cloud Storage location where the output is to - be written to. In the given directory a new directory will be - created with name: - ``export-data--`` - where timestamp is in YYYYMMDDHHMMSS format. All export - output will be written into that directory. Inside that - directory, annotations with the same schema will be grouped - into sub directories which are named with the corresponding - annotations' schema title. Inside these sub directories, a - schema.yaml will be created to describe the output format. - If the uri doesn't end with '/', a '/' will be automatically - appended. The directory is created if it doesn't exist. - project: Project to retrieve Dataset from. + output_dir: The Google Cloud Storage location where the output is to be written to. In the given directory a new directory will be created with name: `export-data--` where timestamp is in YYYYMMDDHHMMSS format. All export output will be written into that directory. Inside that directory, annotations with the same schema will be grouped into sub directories which are named with the corresponding annotations' schema title. Inside these sub directories, a schema.yaml will be created to describe the output format. If the uri doesn't end with '/', a '/' will be automatically appended. The directory is created if it doesn't exist. location: Optional location to retrieve Dataset from. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: exported_dataset: All of the files that are exported in this export operation. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_text_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_text_dataset/component.py index 5c453264e5..e3a0e9956c 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_text_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_text_dataset/component.py @@ -15,6 +15,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Input @@ -23,30 +24,19 @@ @dsl.container_component def text_dataset_export( - project: str, dataset: Input[VertexDataset], output_dir: str, exported_dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Exports `Dataset `_ to a GCS output directory. + """Exports [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) to a GCS output directory. Args: - output_dir: The Google Cloud Storage location where the output is to - be written to. In the given directory a new directory will be - created with name: - ``export-data--`` - where timestamp is in YYYYMMDDHHMMSS format. All export - output will be written into that directory. Inside that - directory, annotations with the same schema will be grouped - into sub directories which are named with the corresponding - annotations' schema title. Inside these sub directories, a - schema.yaml will be created to describe the output format. - If the uri doesn't end with '/', a '/' will be automatically - appended. The directory is created if it doesn't exist. - project: Project to retrieve Dataset from. + output_dir: The Google Cloud Storage location where the output is to be written to. In the given directory a new directory will be created with name: `export-data--` where timestamp is in YYYYMMDDHHMMSS format. All export output will be written into that directory. Inside that directory, annotations with the same schema will be grouped into sub directories which are named with the corresponding annotations' schema title. Inside these sub directories, a schema.yaml will be created to describe the output format. If the uri doesn't end with '/', a '/' will be automatically appended. The directory is created if it doesn't exist. location: Optional location to retrieve Dataset from. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: exported_dataset: All of the files that are exported in this export operation. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_time_series_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_time_series_dataset/component.py index d186adc71a..144770ddcb 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_time_series_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_time_series_dataset/component.py @@ -15,6 +15,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Input @@ -23,30 +24,19 @@ @dsl.container_component def time_series_dataset_export( - project: str, dataset: Input[VertexDataset], output_dir: str, exported_dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Exports `Dataset `_ to a GCS output directory. + """Exports [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) to a GCS output directory. Args: - output_dir: The Google Cloud Storage location where the output is to - be written to. In the given directory a new directory will be - created with name: - ``export-data--`` - where timestamp is in YYYYMMDDHHMMSS format. All export - output will be written into that directory. Inside that - directory, annotations with the same schema will be grouped - into sub directories which are named with the corresponding - annotations' schema title. Inside these sub directories, a - schema.yaml will be created to describe the output format. - If the uri doesn't end with '/', a '/' will be automatically - appended. The directory is created if it doesn't exist. - project: Project to retrieve Datasetfrom. + output_dir: The Google Cloud Storage location where the output is to be written to. In the given directory a new directory will be created with name: `export-data--` where timestamp is in YYYYMMDDHHMMSS format. All export output will be written into that directory. Inside that directory, annotations with the same schema will be grouped into sub directories which are named with the corresponding annotations' schema title. Inside these sub directories, a schema.yaml will be created to describe the output format. If the uri doesn't end with '/', a '/' will be automatically appended. The directory is created if it doesn't exist. location: Optional location to retrieve Datasetfrom. + project: Project to retrieve Datasetfrom. Defaults to the project in which the PipelineJob is run. Returns: exported_dataset: All of the files that are exported in this export operation. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_video_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_video_dataset/component.py index 6f73809245..4fb30c8e10 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_video_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/export_video_dataset/component.py @@ -15,6 +15,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl from kfp.dsl import Input @@ -23,30 +24,19 @@ @dsl.container_component def video_dataset_export( - project: str, dataset: Input[VertexDataset], output_dir: str, exported_dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Exports `Dataset `_ to a GCS output directory. + """Exports [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) to a GCS output directory. Args: - output_dir: The Google Cloud Storage location where the output is to - be written to. In the given directory a new directory will be - created with name: - ``export-data--`` - where timestamp is in YYYYMMDDHHMMSS format. All export - output will be written into that directory. Inside that - directory, annotations with the same schema will be grouped - into sub directories which are named with the corresponding - annotations' schema title. Inside these sub directories, a - schema.yaml will be created to describe the output format. - If the uri doesn't end with '/', a '/' will be automatically - appended. The directory is created if it doesn't exist. - project: Project to retrieve Dataset from. + output_dir: The Google Cloud Storage location where the output is to be written to. In the given directory a new directory will be created with name: `export-data--` where timestamp is in YYYYMMDDHHMMSS format. All export output will be written into that directory. Inside that directory, annotations with the same schema will be grouped into sub directories which are named with the corresponding annotations' schema title. Inside these sub directories, a schema.yaml will be created to describe the output format. If the uri doesn't end with '/', a '/' will be automatically appended. The directory is created if it doesn't exist. location: Optional location to retrieve Dataset from. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: exported_dataset: All of the files that are exported in this export operation. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/get_vertex_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/get_vertex_dataset/component.py index 801ec50133..4686a1162b 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/get_vertex_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/get_vertex_dataset/component.py @@ -25,13 +25,13 @@ def get_vertex_dataset( gcp_resources: dsl.OutputPath(str), ): # fmt: off - """Gets a `Dataset `_ artifact as a Vertex Dataset artifact. + """Gets a [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets) artifact as a Vertex Dataset artifact. Args: - dataset_resource_name: Vertex Dataset resource name in the format of ``projects/{project}/locations/{location}/datasets/{dataset}``. + dataset_resource_name: Vertex Dataset resource name in the format of `projects/{project}/locations/{location}/datasets/{dataset}`. Returns: - dataset: Vertex Dataset artifact with a ``resourceName`` metadata field in the format of ``projects/{project}/locations/{location}/datasets/{dataset}``. + dataset: Vertex Dataset artifact with a `resourceName` metadata field in the format of `projects/{project}/locations/{location}/datasets/{dataset}`. """ # fmt: on diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_image_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_image_dataset/component.py index 9dc153bea7..cfe3c35715 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_image_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_image_dataset/component.py @@ -16,6 +16,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components import utils from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl @@ -26,44 +27,24 @@ @utils.gcpc_output_name_converter('dataset') @dsl.container_component def image_dataset_import( - project: str, dataset: Input[VertexDataset], output__dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', data_item_labels: Optional[dict] = {}, gcs_source: Optional[str] = None, import_schema_uri: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Uploads data to an existing managed `Dataset `_. + """Uploads data to an existing managed [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets). Args: - project: Project to retrieve Dataset from. location: Optional location to retrieve Dataset from. dataset: The Dataset to be updated. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, "gs://bucket/file.csv" or ["gs://bucket/file1.csv", "gs://bucket/file2.csv"]. - import_schema_uri: Points to a YAML file stored on Google Cloud - Storage describing the import format. Validation will be - done against the schema. The schema is defined as an - `OpenAPI 3.0.2 Schema Object `_. - data_item_labels: Labels that will be applied to newly imported DataItems. If - an identical DataItem as one being imported already exists - in the Dataset, then these labels will be appended to these - of the already existing one, and if labels with identical - key is imported before, the old label value will be - overwritten. If two DataItems are identical in the same - import data operation, the labels will be combined and if - key collision happens in this case, one of the values will - be picked randomly. Two DataItems are considered identical - if their content bytes are identical (e.g. image bytes or - pdf bytes). These labels will be overridden by Annotation - labels specified inside index file refenced by - ``import_schema_uri``, e.g. jsonl file. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, "gs://bucket/file.csv" or ["gs://bucket/file1.csv", "gs://bucket/file2.csv"]. + import_schema_uri: Points to a YAML file stored on Google Cloud Storage describing the import format. Validation will be done against the schema. The schema is defined as an [OpenAPI 3.0.2 Schema Object](https://tinyurl.com/y538mdwt). + data_item_labels: Labels that will be applied to newly imported DataItems. If an identical DataItem as one being imported already exists in the Dataset, then these labels will be appended to these of the already existing one, and if labels with identical key is imported before, the old label value will be overwritten. If two DataItems are identical in the same import data operation, the labels will be combined and if key collision happens in this case, one of the values will be picked randomly. Two DataItems are considered identical if their content bytes are identical (e.g. image bytes or pdf bytes). These labels will be overridden by Annotation labels specified inside index file refenced by `import_schema_uri`, e.g. jsonl file. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: dataset: Instantiated representation of the managed Dataset resource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_text_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_text_dataset/component.py index dce6754955..c43497437c 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_text_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_text_dataset/component.py @@ -15,6 +15,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components import utils from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl @@ -25,46 +26,24 @@ @utils.gcpc_output_name_converter('dataset') @dsl.container_component def text_dataset_import( - project: str, dataset: Input[VertexDataset], output__dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', data_item_labels: Optional[dict] = {}, gcs_source: Optional[str] = None, import_schema_uri: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Uploads data to an existing managed `Dataset `_. + """Uploads data to an existing managed [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets). Args: - project: Project to retrieve Datasetfrom. location: Optional location to retrieve Datasetfrom. dataset: The Datasetto be updated. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, "gs://bucket/file.csv" or ["gs://bucket/file1.csv", "gs://bucket/file2.csv"]. - import_schema_uri: Points to a YAML file stored on Google Cloud - Storage describing the import format. Validation will be - done against the schema. The schema is defined as an - `OpenAPI 3.0.2 Schema - Object `_. - data_item_labels: Labels that will be applied to newly imported DataItems. If - an identical DataItem as one being imported already exists - in the Dataset, then these labels will be appended to these - of the already existing one, and if labels with identical - key is imported before, the old label value will be - overwritten. If two DataItems are identical in the same - import data operation, the labels will be combined and if - key collision happens in this case, one of the values will - be picked randomly. Two DataItems are considered identical - if their content bytes are identical (e.g. image bytes or - pdf bytes). These labels will be overridden by Annotation - labels specified inside index file refenced by - ``import_schema_uri``, - e.g. jsonl file. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, "gs://bucket/file.csv" or ["gs://bucket/file1.csv", "gs://bucket/file2.csv"]. + import_schema_uri: Points to a YAML file stored on Google Cloud Storage describing the import format. Validation will be done against the schema. The schema is defined as an [OpenAPI 3.0.2 Schema Object](https://tinyurl.com/y538mdwt). + data_item_labels: Labels that will be applied to newly imported DataItems. If an identical DataItem as one being imported already exists in the Dataset, then these labels will be appended to these of the already existing one, and if labels with identical key is imported before, the old label value will be overwritten. If two DataItems are identical in the same import data operation, the labels will be combined and if key collision happens in this case, one of the values will be picked randomly. Two DataItems are considered identical if their content bytes are identical (e.g. image bytes or pdf bytes). These labels will be overridden by Annotation labels specified inside index file refenced by `import_schema_uri`, e.g. jsonl file. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: dataset: Instantiated representation of the managed Datasetresource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_video_dataset/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_video_dataset/component.py index 920c20862c..b6c5a83281 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_video_dataset/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/dataset/import_video_dataset/component.py @@ -16,6 +16,7 @@ from typing import Optional from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components import utils from google_cloud_pipeline_components.types.artifact_types import VertexDataset from kfp import dsl @@ -26,46 +27,24 @@ @utils.gcpc_output_name_converter('dataset') @dsl.container_component def video_dataset_import( - project: str, dataset: Input[VertexDataset], output__dataset: Output[VertexDataset], location: Optional[str] = 'us-central1', data_item_labels: Optional[dict] = {}, gcs_source: Optional[str] = None, import_schema_uri: Optional[str] = None, + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Uploads data to an existing managed `Dataset `_. + """Uploads data to an existing managed [Dataset](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets). Args: - project: Project to retrieve Dataset from. location: Optional location to retrieve Dataset from. dataset: The Dataset to be updated. - gcs_source: - Google Cloud Storage URI(-s) to the - input file(s). May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. - For example, "gs://bucket/file.csv" or ["gs://bucket/file1.csv", "gs://bucket/file2.csv"]. - import_schema_uri: Points to a YAML file stored on Google Cloud - Storage describing the import format. Validation will be - done against the schema. The schema is defined as an - `OpenAPI 3.0.2 Schema - Object `_. - data_item_labels: Labels that will be applied to newly imported DataItems. If - an identical DataItem as one being imported already exists - in the Dataset, then these labels will be appended to these - of the already existing one, and if labels with identical - key is imported before, the old label value will be - overwritten. If two DataItems are identical in the same - import data operation, the labels will be combined and if - key collision happens in this case, one of the values will - be picked randomly. Two DataItems are considered identical - if their content bytes are identical (e.g. image bytes or - pdf bytes). These labels will be overridden by Annotation - labels specified inside index file refenced by - ``import_schema_uri``, - e.g. jsonl file. + gcs_source: Google Cloud Storage URI(-s) to the input file(s). May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For example, "gs://bucket/file.csv" or ["gs://bucket/file1.csv", "gs://bucket/file2.csv"]. + import_schema_uri: Points to a YAML file stored on Google Cloud Storage describing the import format. Validation will be done against the schema. The schema is defined as an [OpenAPI 3.0.2 Schema Object](https://tinyurl.com/y538mdwt). + data_item_labels: Labels that will be applied to newly imported DataItems. If an identical DataItem as one being imported already exists in the Dataset, then these labels will be appended to these of the already existing one, and if labels with identical key is imported before, the old label value will be overwritten. If two DataItems are identical in the same import data operation, the labels will be combined and if key collision happens in this case, one of the values will be picked randomly. Two DataItems are considered identical if their content bytes are identical (e.g. image bytes or pdf bytes). These labels will be overridden by Annotation labels specified inside index file refenced by `import_schema_uri`, e.g. jsonl file. + project: Project to retrieve Dataset from. Defaults to the project in which the PipelineJob is run. Returns: dataset: Instantiated representation of the managed Dataset resource. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/__init__.py index 18716819c8..ad76f4ebac 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Manage model serving endpoints via `Vertex AI Endpoints `_.""" +# fmt: off +"""Manage model serving endpoints via [Vertex AI Endpoints](https://cloud.google.com/vertex-ai/docs/predictions/overview?_ga=2.161419069.-1686833729.1684288907#model_deployment).""" +# fmt: on from google_cloud_pipeline_components.v1.endpoint.create_endpoint.component import endpoint_create as EndpointCreateOp from google_cloud_pipeline_components.v1.endpoint.delete_endpoint.component import endpoint_delete as EndpointDeleteOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/create_endpoint/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/create_endpoint/component.py index be799c7238..e25e24a996 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/create_endpoint/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/create_endpoint/component.py @@ -15,6 +15,7 @@ from typing import Dict from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import VertexEndpoint from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component @@ -25,7 +26,6 @@ @container_component def endpoint_create( - project: str, display_name: str, gcp_resources: OutputPath(str), endpoint: Output[VertexEndpoint], @@ -34,45 +34,23 @@ def endpoint_create( labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', network: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """`Creates `_ a Google Cloud Vertex `Endpoint `_ and waits for it to be ready. - - - See the `Endpoint create `_ method for more information. + """[Creates](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/create) a Google Cloud Vertex [Endpoint](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints) and waits for it to be ready. See the [Endpoint create](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/create) method for more information. Args: - project: Project to create the Endpoint. - location: Location to create the Endpoint. If not set, - default to us-central1. - display_name: The user-defined name of the Endpoint. The - name can be up to 128 characters long and can be consist of any UTF-8 - characters. + location: Location to create the Endpoint. If not set, default to us-central1. + display_name: The user-defined name of the Endpoint. The name can be up to 128 characters long and can be consist of any UTF-8 characters. description: The description of the Endpoint. - labels: The labels with user-defined metadata to - organize your Endpoints. Label keys and values can be no longer than - 64 characters (Unicode codepoints), can only contain lowercase - letters, numeric characters, underscores and dashes. International - characters are allowed. See https://goo.gl/xmQnxf for more - information and examples of labels. - encryption_spec_key_name: Customer-managed encryption - key spec for an Endpoint. If set, this Endpoint and all of this - Endoint's sub-resources will be secured by this key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource - is created. If set, this Endpoint and all sub-resources of this - Endpoint will be secured by this key. - network: The full name of the Google Compute Engine - network to which the Endpoint should be peered. Private services - access must already be configured for the network. If left - unspecified, the Endpoint is not peered with any network. - `Format `_: - ``projects/{project}/global/networks/{network}``. Where ``{project}`` is a - project number, as in ``'12345'``, and ``{network}`` is network name. + labels: The labels with user-defined metadata to organize your Endpoints. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + encryption_spec_key_name: Customer-managed encryption key spec for an Endpoint. If set, this Endpoint and all of this Endoint's sub-resources will be secured by this key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. If set, this Endpoint and all sub-resources of this Endpoint will be secured by this key. + network: The full name of the Google Compute Engine network to which the Endpoint should be peered. Private services access must already be configured for the network. If left unspecified, the Endpoint is not peered with any network. [Format](https://cloud.google.com/compute/docs/reference/rest/v1/networks/insert): `projects/{project}/global/networks/{network}`. Where `{project}` is a project number, as in `'12345'`, and `{network}` is network name. + project: Project to create the Endpoint. Defaults to the project in which the PipelineJob is run. Returns: endpoint: Artifact tracking the created Endpoint. - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the create Endpoint's long-running operation. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the create Endpoint's long-running operation. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/delete_endpoint/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/delete_endpoint/component.py index d8bf307f7e..6ee491e180 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/delete_endpoint/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/delete_endpoint/component.py @@ -25,15 +25,13 @@ def endpoint_delete( gcp_resources: dsl.OutputPath(str), ): # fmt: off - """`Deletes `_ a Google Cloud Vertex `Endpoint `_. - - See the `Endpoint delete `_ method for more information. + """[Deletes](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/delete) a Google Cloud Vertex [Endpoint](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints). See the [Endpoint delete](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/delete) method for more information. Args: endpoint: The Endpoint to be deleted. Returns: - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the delete Endpoint's long-running operation. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the delete Endpoint's long-running operation. """ # fmt: on return dsl.ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/deploy_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/deploy_model/component.py index 9136fdb589..5cc5fa78f2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/deploy_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/deploy_model/component.py @@ -45,95 +45,28 @@ def model_deploy( explanation_parameters: Dict[str, str] = {}, ): # fmt: off - """`Deploys `_ a Google Cloud Vertex Model to an `Endpoint `_ creating a - `DeployedModel `_ within it. - - See the `deploy Model `_ method for more information. + """[Deploys](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/deployModel) a Google Cloud Vertex Model to an [Endpoint](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints) creating a [DeployedModel](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints#deployedmodel) within it. See the [deploy Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/deployModel) method for more information. Args: model: The model to be deployed. - endpoint: The Endpoint to be deployed - to. - deployed_model_display_name: The display name of the - DeployedModel. If not provided upon creation, the Model's display_name - is used. - traffic_split: - A map from a DeployedModel's - ID to the percentage of this Endpoint's traffic that should be - forwarded to that DeployedModel. If this field is non-empty, then the - Endpoint's trafficSplit will be overwritten with it. To refer to the - ID of the just being deployed Model, a "0" should be used, and the - actual ID of the new DeployedModel will be filled in its place by this - method. The traffic percentage values must add up to 100. If this - field is empty, then the Endpoint's trafficSplit is not updated. - dedicated_resources_machine_type: The specification of a - single machine used by the prediction. This field is required if - ``automatic_resources_min_replica_count`` is not specified. See `more information `_. - dedicated_resources_accelerator_type: Hardware - accelerator type. Must also set accelerator_count if used. See `available options `_. This field is required if - ``dedicated_resources_machine_type`` is specified. - dedicated_resources_accelerator_count: The number of - accelerators to attach to a worker replica. - dedicated_resources_min_replica_count: The minimum - number of machine replicas this DeployedModel will be always deployed - on. This value must be greater than or equal to 1. If traffic against - the DeployedModel increases, it may dynamically be deployed onto more - replicas, and as traffic decreases, some of these extra replicas may - be freed. - dedicated_resources_max_replica_count: The maximum - number of replicas this deployed model may the larger value of - min_replica_count or 1 will be used. If value provided is smaller than - min_replica_count, it will automatically be increased to be - min_replica_count. The maximum number of replicas this deployed model - may be deployed on when the traffic against it increases. If requested - value is too large, the deployment will error, but if deployment - succeeds then the ability to scale the model to that many replicas is - guaranteed (barring service outages). If traffic against the deployed - model increases beyond what its replicas at maximum may handle, a - portion of the traffic will be dropped. If this value is not provided, - will use ``dedicated_resources_min_replica_count`` as the default value. - automatic_resources_min_replica_count: The minimum - number of replicas this DeployedModel will be always deployed on. If - traffic against it increases, it may dynamically be deployed onto more - replicas up to ``automatic_resources_max_replica_count``, and as traffic - decreases, some of these extra replicas may be freed. If the requested - value is too large, the deployment will error. This field is required - if ``dedicated_resources_machine_type`` is not specified. - automatic_resources_max_replica_count: The maximum - number of replicas this DeployedModel may be deployed on when the - traffic against it increases. If the requested value is too large, the - deployment will error, but if deployment succeeds then the ability to - scale the model to that many replicas is guaranteed (barring service - outages). If traffic against the DeployedModel increases beyond what - its replicas at maximum may handle, a portion of the traffic will be - dropped. If this value is not provided, a no upper bound for scaling - under heavy traffic will be assume, though Vertex AI may be unable to - scale beyond certain replica number. - service_account: The service account that the - DeployedModel's container runs as. Specify the email address of the - service account. If this service account is not specified, the - container runs as a service account that doesn't have access to the - resource project. Users deploying the Model must have the - ``iam.serviceAccounts.actAs`` permission on this service account. - disable_container_logging: For custom-trained Models - and AutoML Tabular Models, the container of the DeployedModel - instances will send stderr and stdout streams to Stackdriver Logging - by default. Please note that the logs incur cost, which are subject to - Cloud Logging pricing. User can disable container logging by setting - this flag to true. - enable_access_logging: These logs are like standard - server access logs, containing information like timestamp and latency - for each prediction request. Note that Stackdriver logs may incur a - cost, especially if your project receives prediction requests at a - high queries per second rate (QPS). Estimate your costs before - enabling this option. - explanation_metadata: Metadata describing the Model's - input and output for explanation. See `more information `_. - explanation_parameters: Parameters that configure - explaining information of the Model's predictions. See `more information `_. + endpoint: The Endpoint to be deployed to. + deployed_model_display_name: The display name of the DeployedModel. If not provided upon creation, the Model's display_name is used. + traffic_split: A map from a DeployedModel's ID to the percentage of this Endpoint's traffic that should be forwarded to that DeployedModel. If this field is non-empty, then the Endpoint's trafficSplit will be overwritten with it. To refer to the ID of the just being deployed Model, a "0" should be used, and the actual ID of the new DeployedModel will be filled in its place by this method. The traffic percentage values must add up to 100. If this field is empty, then the Endpoint's trafficSplit is not updated. + dedicated_resources_machine_type: The specification of a single machine used by the prediction. This field is required if `automatic_resources_min_replica_count` is not specified. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints#dedicatedresources). + dedicated_resources_accelerator_type: Hardware accelerator type. Must also set accelerator_count if used. See [available options](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec#AcceleratorType). This field is required if `dedicated_resources_machine_type` is specified. + dedicated_resources_accelerator_count: The number of accelerators to attach to a worker replica. + dedicated_resources_min_replica_count: The minimum number of machine replicas this DeployedModel will be always deployed on. This value must be greater than or equal to 1. If traffic against the DeployedModel increases, it may dynamically be deployed onto more replicas, and as traffic decreases, some of these extra replicas may be freed. + dedicated_resources_max_replica_count: The maximum number of replicas this deployed model may the larger value of min_replica_count or 1 will be used. If value provided is smaller than min_replica_count, it will automatically be increased to be min_replica_count. The maximum number of replicas this deployed model may be deployed on when the traffic against it increases. If requested value is too large, the deployment will error, but if deployment succeeds then the ability to scale the model to that many replicas is guaranteed (barring service outages). If traffic against the deployed model increases beyond what its replicas at maximum may handle, a portion of the traffic will be dropped. If this value is not provided, will use `dedicated_resources_min_replica_count` as the default value. + automatic_resources_min_replica_count: The minimum number of replicas this DeployedModel will be always deployed on. If traffic against it increases, it may dynamically be deployed onto more replicas up to `automatic_resources_max_replica_count`, and as traffic decreases, some of these extra replicas may be freed. If the requested value is too large, the deployment will error. This field is required if `dedicated_resources_machine_type` is not specified. + automatic_resources_max_replica_count: The maximum number of replicas this DeployedModel may be deployed on when the traffic against it increases. If the requested value is too large, the deployment will error, but if deployment succeeds then the ability to scale the model to that many replicas is guaranteed (barring service outages). If traffic against the DeployedModel increases beyond what its replicas at maximum may handle, a portion of the traffic will be dropped. If this value is not provided, a no upper bound for scaling under heavy traffic will be assume, though Vertex AI may be unable to scale beyond certain replica number. + service_account: The service account that the DeployedModel's container runs as. Specify the email address of the service account. If this service account is not specified, the container runs as a service account that doesn't have access to the resource project. Users deploying the Model must have the `iam.serviceAccounts.actAs` permission on this service account. + disable_container_logging: For custom-trained Models and AutoML Tabular Models, the container of the DeployedModel instances will send stderr and stdout streams to Stackdriver Logging by default. Please note that the logs incur cost, which are subject to Cloud Logging pricing. User can disable container logging by setting this flag to true. + enable_access_logging: These logs are like standard server access logs, containing information like timestamp and latency for each prediction request. Note that Stackdriver logs may incur a cost, especially if your project receives prediction requests at a high queries per second rate (QPS). Estimate your costs before enabling this option. + explanation_metadata: Metadata describing the Model's input and output for explanation. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata). + explanation_parameters: Parameters that configure explaining information of the Model's predictions. See [more information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata). Returns: - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the deploy Model's long-running operation. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the deploy Model's long-running operation. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/undeploy_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/undeploy_model/component.py index ee28438485..6c2876060f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/undeploy_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/endpoint/undeploy_model/component.py @@ -30,23 +30,15 @@ def model_undeploy( traffic_split: Dict[str, str] = {}, ): # fmt: off - """`Undeploys `_ a Google Cloud Vertex `DeployedModel `_ within an `Endpoint `_. - - See the `undeploy Model `_ method for more information. - + """[Undeploys](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/undeployModel) a Google Cloud Vertex [DeployedModel](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints#deployedmodel) within an [Endpoint](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints). See the [undeploy Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.endpoints/undeployModel) method for more information. Args: model: The model that was deployed to the Endpoint. endpoint: The Endpoint for the DeployedModel to be undeployed from. - traffic_split: - If this field is provided, then the Endpoint's trafficSplit will be overwritten with it. - If last DeployedModel is being undeployed from the Endpoint, the - [Endpoint.traffic_split] will always end up empty when this call returns. - A DeployedModel will be successfully undeployed only if it doesn't have any traffic - assigned to it when this method executes, or if this field unassigns any traffic to it. + traffic_split: If this field is provided, then the Endpoint's trafficSplit will be overwritten with it. If last DeployedModel is being undeployed from the Endpoint, the [Endpoint.traffic_split] will always end up empty when this call returns. A DeployedModel will be successfully undeployed only if it doesn't have any traffic assigned to it when this method executes, or if this field unassigns any traffic to it. Returns: - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the undeploy Model's long-running operation. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the undeploy Model's long-running operation. """ # fmt: on return dsl.ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/README.md b/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/README.md deleted file mode 100644 index b28f8f8fdc..0000000000 --- a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/README.md +++ /dev/null @@ -1,152 +0,0 @@ -# Forecasting Components Inputs - -## input_tables - -**input_tables** is a seriazlied JSON array required by both ForecastingPreprocessingOp and ForecastingValidationOp. - -Proto definition of TableSpecs: -```protobuf -// Desribes a BigQuery user input table for Vertex AI validation, preprocessing -// and training. -message TableSpecs { - // [Required] BigQuery table path of the table. e.g.: - // bq://projectId.datasetId.tableId - string bigquery_uri = 1; - - // [Required] The table type from the eligible types: FORECASTING_PRIMARY, - // FORECASTING_ATTRIBUTE and FORECASTING_PLAN - string table_type = 2; - - // Some table types require additional information about the table. If - // table_type is FORECASTING_PRIMARY, forecasting_primary_table_metadata is - // required. If table_type is FORECASTING_ATTRIBUTE, - // forecasting_attribute_table_metadata is required. - oneof metadata { - ForecastingPrimaryTableMetadata forecasting_primary_table_metadata = 3; - ForecastingAttributeTableMetadata forecasting_attribute_table_metadata = 4; - } -} - -// The metadata that desribes the primary table in Vertex forecasting. -// -// The primary table must contain historical data at the granularity it will -// predict at. For example, if the task is to predict daily sales, this table -// should have a target column with historical sales data at daily granularity. -// -// One or more time series identifier columns are needed in this table. If this -// table has 2 time series identifier columns - "channel_id" and "product_id", -// a time series will be identified by the combination of these 2 columns. -// -// A time column must be present in this table with DATE, DATETIME or TIMESTAMP -// type that reflects the specified granularity. -// -// Two rows cannot have the same value in both the time column and the time -// series identifier column(s). -// -// Except for the time series identifier column(s), every column in the -// primary table will be considered time variant. For example, a holiday -// column or promotion column could have different values at different time -// given a specific time series identifier. If a column has fixed value given a -// time series identifier, i.e. the color of a product given the product ID as -// time series identifier, the column should be moved to the attribute table. -message ForecastingPrimaryTableMetadata { - // [Required] The name of the column that identifies time order in the time - // series. - string time_column = 1; - // [Required] The name of the column that the model is to predict. - string target_column = 2; - // [Required] Names of columns that jointly identify the time series. - repeated string time_series_identifier_columns = 3; - // [Optional] Names of columns that are unavailable when a forecast is - // requested. This column contains information for the given entity - // (identified by the time_series_identifier_columns) that is unknown before - // the forecast For example, actual weather on a given day. - repeated string unavailable_at_forecast_columns = 4; - - // [Required] The granularity of time units presented in the time_column. - TimeGranularity time_granularity = 5; - // [Optional] The name of the column that splits the table. Eligible values - // are: TRAIN, VALIDATE, TEST - string predefined_splits_column = 6; - // [Optional] The name of the column that measures the importance of the row. - // Higher weight values give more importance to the corresponding row during - // model training. For example, to let the model pay more attention to - // holidays, the holiday rows can have weight value 1.0 and the rest rows have - // a weight value 0.5. Weight value must be >= 0. 0 means ignored in training, - // validation or test. If not specified, all rows will have an equal weight of - // 1. - string weight_column = 7; -} - -// A duration of time expressed in time granularity units. -message TimeGranularity { - // [Required] The unit of this time period. Eligible values are: MINUTE, HOUR, - // DAY, WEEK, MONTH, YEAR - string unit = 1; - // [Required] The number of units per period, e.g. 3 weeks or 2 months. - int64 quantity = 2; -} - -// The metadata that desribes the attribute table in Vertex forecasting. -// -// Attribute table contains features that desribe time series that are not -// changed with time. For example, if the primary table has 2 -// time_series_identifier_columns columns - product_id and channel_id, an -// optional attribute table can provide product attributes such as category, -// color etc. -// -// The attribute table should have a single identifier column that is the same -// as one of the time_series_identifier_columns in -// ForecastingPrimaryTableMetadata. -message ForecastingAttributeTableMetadata { - // [Required] The name of the primary key column. - string primary_key_column = 1; -} -``` - -# Example pipeline using ForecastingPreprocessingOp and ForecastingValidationOp -```python -import json -from google_cloud_pipeline_components.v1 import forecasting - - -primary_table_specs = { - "bigquery_uri": "bq://endless-forms-most-beautiful.iowa_liquor_sales_forecast.sales_table", - "table_type": "FORECASTING_PRIMARY", - "forecasting_primary_table_metadata": { - "time_column": "datetime", - "target_column": "gross_quantity", - "time_series_identifier_columns": ["product_id", "location_id"], - "unavailable_at_forecast_columns": ['sale_dollars', 'state_bottle_cost', 'state_bottle_retail'], - "time_granularity": {"unit": "DAY", "quantity": 1 }, - "predefined_splits_column": "ml_use" - } -} - -attribute_table_specs1 = { - "bigquery_uri": "bq://endless-forms-most-beautiful.iowa_liquor_sales_forecast.product_table", - "table_type": "FORECASTING_ATTRIBUTE", - "forecasting_attribute_table_metadata": { - "primary_key_column": "product_id" - } -} - -attribute_table_specs2 = { - "bigquery_uri": "bq://endless-forms-most-beautiful.iowa_liquor_sales_forecast.location_table", - "table_type": "FORECASTING_ATTRIBUTE", - "forecasting_attribute_table_metadata": { - "primary_key_column": "location_id" - } -} - -input_table_specs = [primary_table_specs, attribute_table_specs1, attribute_table_specs2] -input_tables = json.dumps(input_table_specs) - - -@dsl.pipeline(name='forecasting-pipeline-training') -def pipeline(input_tables: str): - # A workflow consists of training validation and preprocessing: - validation = forecasting.ForecastingValidationOp(input_tables=input_tables, validation_theme='FORECASTING_TRAINING') - preprocess = forecasting.ForecastingPreprocessingOp(project_id='endless-forms-most-beautiful', input_tables=input_tables) - preprocess.after(validation) -``` diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/__init__.py index fddf25bf11..5ef59e4403 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Compose `tabular data forecasting `_ pipelines.""" +# fmt: off +"""Compose [tabular data forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview) pipelines.""" +# fmt: on from google_cloud_pipeline_components.v1.forecasting.prepare_data_for_train.component import prepare_data_for_train as ForecastingPrepareDataForTrainOp from google_cloud_pipeline_components.v1.forecasting.preprocess.component import forecasting_preprocessing as ForecastingPreprocessingOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/prepare_data_for_train/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/prepare_data_for_train/component.py index b926b852fb..2651515ad0 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/prepare_data_for_train/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/prepare_data_for_train/component.py @@ -48,31 +48,23 @@ def prepare_data_for_train( AutoMLForecastingTrainingJobRunOp. Args: - input_tables: Serialized Json array that specifies - input BigQuery tables and specs. - preprocess_metadata: The output of - ForecastingPreprocessingOp that is a serialized dictionary with 2 fields: - processed_bigquery_table_uri and column_metadata. - model_feature_columns: Serialized list of column names - that will be used as input feature in the training step. If None, all - columns will be used in training. + input_tables: Serialized Json array that specifies input BigQuery tables and specs. + preprocess_metadata: The output of ForecastingPreprocessingOp that is a serialized dictionary with 2 fields: processed_bigquery_table_uri and column_metadata. + model_feature_columns: Serialized list of column names that will be used as input feature in the training step. If None, all columns will be used in training. Returns: - NamedTuple: - time_series_identifier_column: Name of the column that identifies the time series. - time_series_attribute_columns: Serialized column names that should be used as attribute columns. - available_at_forecast_columns: Serialized column names of columns that are available at forecast. - unavailable_at_forecast_columns: Serialized column names of columns that are unavailable at forecast. - column_transformations: Serialized transformations to apply to the input columns. - preprocess_bq_uri: The BigQuery table that saves the preprocessing result and will be - used as training input. - target_column: The name of the column values of which the Model is to predict. - time_column: Name of the column that identifies time order in the time series. - predefined_split_column: Name of the column that specifies an ML use of the row. - weight_column: Name of the column that should be used as the weight column. - data_granularity_unit: The data granularity unit. - data_granularity_count: The number of data granularity units between data points in the - training data. + time_series_identifier_column: Name of the column that identifies the time series. + time_series_attribute_columns: Serialized column names that should be used as attribute columns. + available_at_forecast_columns: Serialized column names of columns that are available at forecast. + unavailable_at_forecast_columns: Serialized column names of columns that are unavailable at forecast. + column_transformations: Serialized transformations to apply to the input columns. + preprocess_bq_uri: The BigQuery table that saves the preprocessing result and will be used as training input. + target_column: The name of the column values of which the Model is to predict. + time_column: Name of the column that identifies time order in the time series. + predefined_split_column: Name of the column that specifies an ML use of the row. + weight_column: Name of the column that should be used as the weight column. + data_granularity_unit: The data granularity unit. + data_granularity_count: The number of data granularity units between data points in the training data. """ # fmt: on # pylint: disable=g-import-not-at-top,import-outside-toplevel,redefined-outer-name,reimported diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/preprocess/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/preprocess/component.py index 3dc7705a3e..8fc49f6e9a 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/preprocess/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/forecasting/preprocess/component.py @@ -37,8 +37,7 @@ def forecasting_preprocessing( Args: project: The GCP project id that runs the pipeline. input_tables: Serialized Json array that specifies input BigQuery tables and specs. - preprocessing_bigquery_dataset: Optional BigQuery dataset to save the preprocessing result BigQuery table. - If not present, a new dataset will be created by the component. + preprocessing_bigquery_dataset: Optional BigQuery dataset to save the preprocessing result BigQuery table. If not present, a new dataset will be created by the component. location: Optional location for the BigQuery data, default is US. Returns: diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/__init__.py index 2098bb8442..6b527c15b1 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/__init__.py @@ -11,14 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Create `hyperparameter tuning jobs `_ via a `Vertex AI Custom Training Job `_.""" - +# fmt: off +"""Create [hyperparameter tuning jobs](https://cloud.google.com/vertex-ai/docs/training/using-hyperparameter-tuning) via a [Vertex AI Custom Training Job](https://cloud.google.com/vertex-ai/docs/training/create-custom-job).""" +# fmt: on from google_cloud_pipeline_components.v1.hyperparameter_tuning_job.component import hyperparameter_tuning_job as HyperparameterTuningJobRunOp -from google_cloud_pipeline_components.v1.hyperparameter_tuning_job.utils import ( - serialize_metrics, - serialize_parameters, -) +from google_cloud_pipeline_components.v1.hyperparameter_tuning_job.utils import serialize_metrics +from google_cloud_pipeline_components.v1.hyperparameter_tuning_job.utils import serialize_parameters __all__ = [ 'HyperparameterTuningJobRunOp', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/component.py index 6f326ddff5..34d0424239 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/component.py @@ -15,6 +15,7 @@ from typing import List from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from kfp.dsl import ConcatPlaceholder from kfp.dsl import container_component from kfp.dsl import ContainerSpec @@ -23,7 +24,6 @@ @container_component def hyperparameter_tuning_job( - project: str, display_name: str, base_output_directory: str, worker_pool_specs: List[str], @@ -39,112 +39,49 @@ def hyperparameter_tuning_job( encryption_spec_key_name: str = '', service_account: str = '', network: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Creates a Vertex AI hyperparameter tuning job and waits for - it to complete. + """Creates a Vertex AI hyperparameter tuning job and waits for it to + complete. - See `more information. `_ + See [more information](https://cloud.google.com/vertex-ai/docs/training/using-hyperparameter-tuning). Args: - display_name: The user-defined name of the - HyperparameterTuningJob. The name can be up to 128 characters long and - can be consist of any UTF-8 characters. - project: Project to run the HyperparameterTuningJob in. - base_output_directory: The Cloud Storage location to - store the output of this HyperparameterTuningJob. The - base_output_directory of each child CustomJob backing a Trial is set - to a subdirectory with name as the trial id under its parent - HyperparameterTuningJob's ``base_output_directory``. The following Vertex - AI environment variables will be passed to containers or Python - modules when this field is set: - * AIP_MODEL_DIR = `\/\/model\/` - * AIP_CHECKPOINT_DIR = `\/\/checkpoints\/` - * AIP_TENSORBOARD_LOG_DIR = `\/\/logs\/` - worker_pool_specs: The spec of the worker pools - including machine type and Docker image. All worker pools except the - first one are optional and can be skipped by providing an empty value. - study_spec_metrics: - List serialized from dictionary representing the metrics to optimize. The dictionary key is the metric_id, which is reported by your training job, and the - dictionary value is the optimization goal of the metric (``'minimize'`` or - ``'maximize'``). + display_name: The user-defined name of the HyperparameterTuningJob. The name can be up to 128 characters long and can be consist of any UTF-8 characters. + base_output_directory: The Cloud Storage location to store the output of this HyperparameterTuningJob. The base_output_directory of each child CustomJob backing a Trial is set to a subdirectory with name as the trial id under its parent HyperparameterTuningJob's `base_output_directory`. The following Vertex AI environment variables will be passed to containers or Python modules when this field is set: * AIP_MODEL_DIR = `\/\/model\/` * AIP_CHECKPOINT_DIR = `\/\/checkpoints\/` * AIP_TENSORBOARD_LOG_DIR = `\/\/logs\/` + worker_pool_specs: The spec of the worker pools including machine type and Docker image. All worker pools except the first one are optional and can be skipped by providing an empty value. + study_spec_metrics: List serialized from dictionary representing the metrics to optimize. The dictionary key is the metric_id, which is reported by your training job, and the dictionary value is the optimization goal of the metric (`'minimize'` or `'maximize'`). Example: - Example: - :: + metrics = hyperparameter_tuning_job.serialize_metrics({ 'loss': 'minimize', 'accuracy': 'maximize' }) - metrics = hyperparameter_tuning_job.serialize_metrics({ - 'loss': 'minimize', - 'accuracy': 'maximize' - }) + study_spec_parameters: List serialized from the parameter dictionary. The dictionary represents parameters to optimize. The dictionary key is the parameter_id, which is passed into your training job as a command line key word argument, and the dictionary value is the parameter specification of the metric. Example: - study_spec_parameters: List serialized from the - parameter dictionary. The dictionary represents parameters to - optimize. The dictionary key is the parameter_id, which is passed into - your training job as a command line key word argument, and the - dictionary value is the parameter specification of the metric. + from google.cloud.aiplatform import hyperparameter_tuning as hpt + from google_cloud_pipeline_components.v1 import hyperparameter_tuning_job + parameters = hyperparameter_tuning_job.serialize_parameters({ 'lr': hpt.DoubleParameterSpec(min=0.001, max=0.1, scale='log'), 'units': hpt.IntegerParameterSpec(min=4, max=128, scale='linear'), 'activation': hpt.CategoricalParameterSpec(values=['relu', 'selu']), 'batch_size': hpt.DiscreteParameterSpec(values=[128, 256], scale='linear') }) - :Example: - :: + Parameters specs should be subclasses of [_ParameterSpec](https://github.com/googleapis/python-aiplatform/blob/1fda4172baaf200414d95e7217bfef0e500cc16a/google/cloud/aiplatform/hyperparameter_tuning.py#L51). Supported subclasses include: `DoubleParameterSpec`, `IntegerParameterSpec`, `CategoricalParameterSpace`, `DiscreteParameterSpec`. - from google.cloud.aiplatform import hyperparameter_tuning as hpt - from google_cloud_pipeline_components.v1 import hyperparameter_tuning_job - - parameters = hyperparameter_tuning_job.serialize_parameters({ - 'lr': hpt.DoubleParameterSpec(min=0.001, max=0.1, scale='log'), - 'units': hpt.IntegerParameterSpec(min=4, max=128, scale='linear'), - 'activation': hpt.CategoricalParameterSpec(values=['relu', 'selu']), - 'batch_size': hpt.DiscreteParameterSpec(values=[128, 256], scale='linear') - }) - - Parameters specs should be subclasses of `_ParameterSpec `_. Supported subclasses include: ``DoubleParameterSpec``, - ``IntegerParameterSpec``, ``CategoricalParameterSpace``, ``DiscreteParameterSpec``. max_trial_count: The desired total number of Trials. - parallel_trial_count: The desired number of Trials to - run in parallel. - max_failed_trial_count: The number of failed Trials that - need to be seen before failing the HyperparameterTuningJob. If set to - 0, Vertex AI decides how many Trials must fail before the whole job - fails. - location: Location to run the HyperparameterTuningJob - in, defaults to ``'us-central1'``. - study_spec_algorithm: The search algorithm specified for - the Study. Accepts one of the following: - * ``'ALGORITHM_UNSPECIFIED'`` - If you do not specify an algorithm, your job uses the default Vertex AI algorithm. The default algorithm applies Bayesian optimization to arrive at the optimal solution with a more effective search over the parameter space. - * ``'GRID_SEARCH'`` - A simple grid search within the feasible space. This option is particularly useful if you want to specify a quantity of trials that is greater than the number of points in the feasible space. In such cases, if you do not specify a grid search, the Vertex AI default algorithm may generate duplicate suggestions. To use grid search, all parameter specs must be of type ``IntegerParameterSpec``, ``CategoricalParameterSpace``, or ``DiscreteParameterSpec``. - * ``'RANDOM_SEARCH'`` - A simple random search within the feasible space. - study_spec_measurement_selection_type: This indicates - which measurement to use if/when the service automatically selects the - final measurement from previously reported intermediate measurements. + parallel_trial_count: The desired number of Trials to run in parallel. + max_failed_trial_count: The number of failed Trials that need to be seen before failing the HyperparameterTuningJob. If set to 0, Vertex AI decides how many Trials must fail before the whole job fails. + location: Location to run the HyperparameterTuningJob in, defaults to `'us-central1'`. + study_spec_algorithm: The search algorithm specified for the Study. Accepts one of the following: + + * `'ALGORITHM_UNSPECIFIED'` - If you do not specify an algorithm, your job uses the default Vertex AI algorithm. The default algorithm applies Bayesian optimization to arrive at the optimal solution with a more effective search over the parameter space. + * `'GRID_SEARCH'` - A simple grid search within the feasible space. This option is particularly useful if you want to specify a quantity of trials that is greater than the number of points in the feasible space. In such cases, if you do not specify a grid search, the Vertex AI default algorithm may generate duplicate suggestions. To use grid search, all parameter specs must be of type `IntegerParameterSpec`, `CategoricalParameterSpace`, or `DiscreteParameterSpec`. + * `'RANDOM_SEARCH'` - A simple random search within the feasible space. - Accepts: ``'BEST_MEASUREMENT'`` or ``'LAST_MEASUREMENT'``. Choose this based on - two considerations: A) Do you expect your measurements to - monotonically improve? If so, choose ``'LAST_MEASUREMENT'``. On the - other hand, if you're in a situation where your system can - "over-train" and you expect the performance to get better for a - while but then start declining, choose ``'BEST_MEASUREMENT'``. B) Are - your measurements significantly noisy and/or irreproducible? If - so, ``'BEST_MEASUREMENT'`` will tend to be over-optimistic, and it may - be better to choose ``'LAST_MEASUREMENT'``. If both or neither of (A) - and (B) apply, it doesn't matter which selection type is chosen. - encryption_spec_key_name: Customer-managed encryption - key options for a HyperparameterTuningJob. If this is set, then all - resources created by the HyperparameterTuningJob will be encrypted - with the provided encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource - is created. - service_account: Specifies the service account for - workload run-as account. Users submitting jobs must have act-as - permission on this run-as account. - network: The full name of the Compute Engine network to - which the job should be peered. For example, - ``projects/12345/global/networks/myVPC``. Private services access must - already be configured for the network. If left unspecified, the job is - not peered with any network. + study_spec_measurement_selection_type: This indicates which measurement to use if/when the service automatically selects the final measurement from previously reported intermediate measurements. Accepts: `'BEST_MEASUREMENT'` or `'LAST_MEASUREMENT'`. Choose this based on two considerations: A) Do you expect your measurements to monotonically improve? If so, choose `'LAST_MEASUREMENT'`. On the other hand, if you're in a situation where your system can "over-train" and you expect the performance to get better for a while but then start declining, choose `'BEST_MEASUREMENT'`. B) Are your measurements significantly noisy and/or irreproducible? If so, `'BEST_MEASUREMENT'` will tend to be over-optimistic, and it may be better to choose `'LAST_MEASUREMENT'`. If both or neither of (A) and (B) apply, it doesn't matter which selection type is chosen. + encryption_spec_key_name: Customer-managed encryption key options for a HyperparameterTuningJob. If this is set, then all resources created by the HyperparameterTuningJob will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + service_account: Specifies the service account for workload run-as account. Users submitting jobs must have act-as permission on this run-as account. + network: The full name of the Compute Engine network to which the job should be peered. For example, `projects/12345/global/networks/myVPC`. Private services access must already be configured for the network. If left unspecified, the job is not peered with any network. + project: Project to run the HyperparameterTuningJob in. Defaults to the project in which the PipelineJob is run. Returns: - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which contains the GCP resource ID of the Hyperparameter Tuning job. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which contains the GCP resource ID of the Hyperparameter Tuning job. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/utils.py b/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/utils.py index 2ce7d5d6eb..fa1d171f74 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/utils.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/hyperparameter_tuning_job/utils.py @@ -14,6 +14,7 @@ """Module for supporting Google Vertex AI Hyperparameter Tuning Job Op.""" from typing import Any, Dict, List + from google.cloud.aiplatform import hyperparameter_tuning from google.cloud.aiplatform_v1.types import study @@ -22,30 +23,16 @@ def serialize_parameters( parameters: Dict[str, hyperparameter_tuning._ParameterSpec] ) -> List[Dict[str, Any]]: # fmt: off - """Utility for converting a hyperparameter tuning `ParameterSpec `_ into a list of dictionaries. + """Utility for converting a hyperparameter tuning [ParameterSpec](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/StudySpec#ParameterSpec) into a list of dictionaries. Args: - parameters (Dict[str, hyperparameter_tuning._ParameterSpec]): Dictionary - of paramater ids to subclasses of `_ParameterSpec `_. Supported subclasses include: ``DoubleParameterSpec``, - ``IntegerParameterSpec``, ``CategoricalParameterSpace``, ``DiscreteParameterSpec``. - - :Example: - :: - - from google.cloud.aiplatform import hyperparameter_tuning as hpt + parameters (Dict[str, hyperparameter_tuning._ParameterSpec]): Dictionary of parameter ids to subclasses of [_ParameterSpec](https://github.com/googleapis/python-aiplatform/blob/1fda4172baaf200414d95e7217bfef0e500cc16a/google/cloud/aiplatform/hyperparameter_tuning.py#L51). Supported subclasses include: `DoubleParameterSpec`, `IntegerParameterSpec`, `CategoricalParameterSpace`, `DiscreteParameterSpec`. Example: - parameters = { - 'decay': - hpt.DoubleParameterSpec(min=1e-7, max=1, scale='linear'), - 'learning_rate': - hpt.DoubleParameterSpec(min=1e-7, max=1, scale='linear'), - 'batch_size': - hpt.DiscreteParamterSpec( - values=[4, 8, 16, 32, 64, 128], scale='linear') - } + from google.cloud.aiplatform import hyperparameter_tuning as hpt + parameters = { 'decay': hpt.DoubleParameterSpec(min=1e-7, max=1, scale='linear'), 'learning_rate': hpt.DoubleParameterSpec(min=1e-7, max=1, scale='linear'), 'batch_size': hpt.DiscreteParamterSpec( values=[4, 8, 16, 32, 64, 128], scale='linear') } Returns: - List of ``ParameterSpec`` dictionaries. + List of `ParameterSpec` dictionaries. """ # fmt: on # the to_dict function is used here instead of the to_json function for compatibility with GAPIC @@ -59,21 +46,14 @@ def serialize_parameters( def serialize_metrics(metric_spec: Dict[str, str]) -> List[Dict[str, Any]]: # fmt: off - """Utility for converting a hyperparameter tuning `MetricSpec `_ into a list of dictionaries. + """Utility for converting a hyperparameter tuning [MetricSpec](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/StudySpec#metricspec) into a list of dictionaries. Args: - metric_spec (Dict[str, str]): Dictionary representing metrics to - optimize. The dictionary key is the metric_id, which is reported by your - training job, and the dictionary value is the optimization goal of the - metric (``'minimize'`` or ``'maximize'``). + metric_spec (Dict[str, str]): Dictionary representing metrics to optimize. The dictionary key is the metric_id, which is reported by your training job, and the dictionary value is the optimization goal of the metric (`'minimize'` or `'maximize'`). Example: - :Example: - :: + metrics = {'loss': 'minimize', 'accuracy': 'maximize'} - metrics = {'loss': 'minimize', 'accuracy': 'maximize'} - - Returns: - List of ``MetricSpec`` dictionaries. + Returns: List of `MetricSpec` dictionaries. """ # fmt: on return [ diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/model/__init__.py index 0ff8cf50df..2295c68d53 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Manage models via `Vertex AI Model Registry `_.""" +# fmt: off +"""Manage models via [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).""" +# fmt: on from google_cloud_pipeline_components.v1.model.delete_model.component import model_delete as ModelDeleteOp from google_cloud_pipeline_components.v1.model.export_model.component import model_export as ModelExportOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model/delete_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/model/delete_model/component.py index 30df2efa52..480c73c749 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model/delete_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model/delete_model/component.py @@ -21,17 +21,13 @@ @dsl.container_component def model_delete(model: Input[VertexModel], gcp_resources: dsl.OutputPath(str)): # fmt: off - """`Deletes `_ a Google Cloud Vertex `Model `_. - - See the `Model delete `_ method for more information. + """[Deletes](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/delete) a Google Cloud Vertex [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models). See the [Model delete](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/delete) method for more information. Note that the full model is deleted, NOT only the model version. Args: - model: The name of the Model resource to be deleted. Format: ``projects/{project}/locations/{location}/models/{model}``. `More information. `_ + model: The name of the Model resource to be deleted. Format: `projects/{project}/locations/{location}/models/{model}`. [More information](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/delete#path-parameters). Returns: - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the delete Model's long-running operation. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the delete Model's long-running operation. """ # fmt: on return dsl.ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model/export_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/model/export_model/component.py index 7cfe4538e4..d184ef385a 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model/export_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model/export_model/component.py @@ -33,47 +33,17 @@ def model_export( image_destination: str = '', ): # fmt: off - """`Exports `_ a Google Cloud Vertex `Model `_ to a user-specified location. - - The Model must be exportable. A Model is considered to be exportable if it has at least one supported - export format. - - See the `Model export `_ method for more information. + """[Exports](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/export) a Google Cloud Vertex [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models) to a user-specified location. The Model must be exportable. A Model is considered to be exportable if it has at least one supported export format. See the [Model export](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/export) method for more information. Args: model: The Model to export. - export_format_id: The ID of the format in which the Model must be - exported. Each Model lists the export formats it supports. If no value - is provided here, then the first from the list of the Model's - supported formats is used by default. `More information. `_ - artifact_destination: The Cloud Storage location where - the Model artifact is to be written to. Under the directory given as - the destination a new one with name - ``"model-export--"``, - where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format, will - be created. Inside, the Model and any of its supporting files will be - written. This field should only be set when, in - [Model.supported_export_formats], the value for the key given in - ``export_format_id`` contains ``ARTIFACT``. `More information. `_ - image_destination: The Google Container Registry or - Artifact Registry URI where the Model container image will be copied - to. `More information. `_ - - Accepted forms: - - - Google Container Registry path. For example: ``gcr.io/projectId/imageName:tag``. - - Artifact Registry path. - - For example: - - ``us-central1-docker.pkg.dev/projectId/repoName/imageName:tag``. - - This field should only be set when, in [Model.supported_export_formats], the value for the key given in ``export_format_id`` contains ``IMAGE``. + export_format_id: The ID of the format in which the Model must be exported. Each Model lists the export formats it supports. If no value is provided here, then the first from the list of the Model's supported formats is used by default. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/export#OutputConfig) + artifact_destination: The Cloud Storage location where the Model artifact is to be written to. Under the directory given as the destination a new one with name `"model-export--"`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format, will be created. Inside, the Model and any of its supporting files will be written. This field should only be set when, in [Model.supported_export_formats], the value for the key given in `export_format_id` contains `ARTIFACT`. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/export#OutputConfig) + image_destination: The Google Container Registry or Artifact Registry URI where the Model container image will be copied to. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/export#OutputConfig) Accepted forms: - Google Container Registry path. For example: `gcr.io/projectId/imageName:tag`. - Artifact Registry path. For example: `us-central1-docker.pkg.dev/projectId/repoName/imageName:tag`. This field should only be set when, in [Model.supported_export_formats], the value for the key given in `export_format_id` contains `IMAGE`. Returns: - output_info: Details of the completed export with output destination paths to - the artifacts or container image. - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the export Model's long-running operation. + output_info: Details of the completed export with output destination paths to the artifacts or container image. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the export Model's long-running operation. """ # fmt: on return ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model/upload_model/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/model/upload_model/component.py index 71ea87e032..7f9397b80a 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model/upload_model/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model/upload_model/component.py @@ -15,6 +15,7 @@ from typing import Dict from google_cloud_pipeline_components import _image +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components.types.artifact_types import UnmanagedContainerModel from google_cloud_pipeline_components.types.artifact_types import VertexModel from kfp import dsl @@ -29,7 +30,6 @@ @container_component def model_upload( - project: str, display_name: str, gcp_resources: OutputPath(str), model: Output[VertexModel], @@ -41,59 +41,32 @@ def model_upload( explanation_parameters: Dict[str, str] = {}, labels: Dict[str, str] = {}, encryption_spec_key_name: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """`Uploads `_ a Google Cloud Vertex `Model `_ and returns a Model artifact representing the uploaded Model - resource. - - See `Model upload `_ method for more information. + """[Uploads](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload) a Google Cloud Vertex [Model](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models) and returns a Model artifact representing the uploaded Model resource, with a tag for the particular version. See [Model upload](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload) method for more information. Args: - project: Project to upload this Model to. - location: Optional location to upload this Model to. If - not set, defaults to ``us-central1``. - display_name: The display name of the Model. The name - can be up to 128 characters long and can be consist of any UTF-8 - characters. `More information. `_ - description: The description of the Model. `More information. `_ - parent_model: An artifact of a model which to upload a new version to. Only specify this field when uploading a new version. `More information. `_ - unmanaged_container_model: The unmanaged container model to be uploaded. The Model can be passed from an upstream step or imported via a KFP ``dsl.importer``. - - :Examples: - :: + location: Optional location to upload this Model to. If not set, defaults to `us-central1`. + display_name: The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + description: The description of the Model. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models#Model) + parent_model: An artifact of a model which to upload a new version to. Only specify this field when uploading a new version. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.models/upload#request-body) + unmanaged_container_model: The unmanaged container model to be uploaded. The Model can be passed from an upstream step or imported via a KFP `dsl.importer`. Example: - from kfp import dsl - from google_cloud_pipeline_components.types import artifact_types + from kfp import dsl + from google_cloud_pipeline_components.types import artifact_types - importer_spec = dsl.importer( - artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model', - artifact_class=artifact_types.UnmanagedContainerModel, - metadata={ - 'containerSpec': { 'imageUri': - 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod' - } - }) + importer_spec = dsl.importer( artifact_uri='gs://managed-pipeline-gcpc-e2e-test/automl-tabular/model', artifact_class=artifact_types.UnmanagedContainerModel, metadata={ 'containerSpec': { 'imageUri': 'us-docker.pkg.dev/vertex-ai/automl-tabular/prediction-server:prod' } }) - explanation_metadata: Metadata describing the Model's - input and output for explanation. Both ``explanation_metadata`` and ``explanation_parameters`` must be passed together when used. `More information. `_ - explanation_parameters: Parameters to configure - explaining for Model's predictions. `More information. `_ - encryption_spec_key_name: Customer-managed encryption - key spec for a Model. If set, this Model and all sub-resources of this - Model will be secured by this key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource - is created. - labels: The labels with user-defined metadata to - organize your model. Label keys and values can be no longer than 64 - characters (Unicode codepoints), can only contain lowercase letters, - numeric characters, underscores and dashes. International characters - are allowed. See https://goo.gl/xmQnxf for more information and - examples of labels. + explanation_metadata: Metadata describing the Model's input and output for explanation. Both `explanation_metadata` and `explanation_parameters` must be passed together when used. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata) + explanation_parameters: Parameters to configure explaining for Model's predictions. [More information.](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters) + encryption_spec_key_name: Customer-managed encryption key spec for a Model. If set, this Model and all sub-resources of this Model will be secured by this key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + labels: The labels with user-defined metadata to organize your model. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. + project: Project to upload this Model to. Defaults to the project in which the PipelineJob is run. Returns: - model: Artifact tracking the created Model. - gcp_resources: Serialized JSON of ``gcp_resources`` `proto `_ which tracks the upload Model's long-running operation. + model: Artifact tracking the created Model version. + gcp_resources: Serialized JSON of `gcp_resources` [proto](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) which tracks the upload Model's long-running operation. """ # fmt: on return ContainerSpec( @@ -142,10 +115,10 @@ def model_upload( '{{$}}', IfPresentPlaceholder( input_name='parent_model', - then=ConcatPlaceholder([ - '--parent_model_name ', + then=[ + '--parent_model_name', parent_model.metadata['resourceName'], - ]), + ], ), ], ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/__init__.py index 7d75f68480..78d839098f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/__init__.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Model evaluation pipelines.""" from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/classification_component.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/classification_component.py index 1c5f24ba35..cbdef55e13 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/classification_component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/classification_component.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, List + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import version from google_cloud_pipeline_components.types.artifact_types import BQTable from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics from google_cloud_pipeline_components.types.artifact_types import VertexModel -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG from kfp import dsl from kfp.dsl import container_component @@ -24,7 +27,6 @@ def model_evaluation_classification( gcp_resources: dsl.OutputPath(str), evaluation_metrics: dsl.Output[ClassificationMetrics], - project: str, target_field_name: str, model: dsl.Input[VertexModel] = None, location: str = 'us-central1', @@ -32,14 +34,14 @@ def model_evaluation_classification( predictions_gcs_source: dsl.Input[dsl.Artifact] = None, predictions_bigquery_source: dsl.Input[BQTable] = None, ground_truth_format: str = 'jsonl', - ground_truth_gcs_source: list = [], + ground_truth_gcs_source: List[str] = [], ground_truth_bigquery_source: str = '', classification_type: str = 'multiclass', - class_labels: list = [], + class_labels: List[str] = [], prediction_score_column: str = 'prediction.scores', prediction_label_column: str = 'prediction.classes', - slicing_specs: list = [], - positive_classes: list = [], + slicing_specs: List[Any] = [], + positive_classes: List[str] = [], dataflow_service_account: str = '', dataflow_disk_size_gb: int = 50, dataflow_machine_type: str = 'n1-standard-4', @@ -49,9 +51,10 @@ def model_evaluation_classification( dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Computes a ``google.ClassificationMetrics`` Artifact, containing evaluation + """Computes a `google.ClassificationMetrics` Artifact, containing evaluation metrics given a model's prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation @@ -60,119 +63,45 @@ def model_evaluation_classification( text data. Args: - project: Project to run evaluation container. location: Location for running the evaluation. - predictions_format: The file format for the batch - prediction results. ``jsonl``, ``csv``, and ``bigquery`` are the allowed - formats, from Vertex Batch Prediction. - predictions_gcs_source: An artifact with its - URI pointing toward a GCS directory with prediction or explanation files - to be used for this evaluation. For prediction results, the files should - be named "prediction.results-*" or "predictions_". For explanation - results, the files should be named "explanation.results-*". - predictions_bigquery_source: BigQuery table - with prediction or explanation data to be used for this evaluation. For - prediction results, the table column should be named "predicted_*". - ground_truth_format: Required for custom tabular and non - tabular data. The file format for the ground truth files. ``jsonl``, - ``csv``, and ``bigquery`` are the allowed formats. - ground_truth_gcs_source: Required for custom - tabular and non tabular data. The GCS URIs representing where the ground - truth is located. Used to provide ground truth for each prediction - instance when they are not part of the batch prediction jobs prediction - instance. - ground_truth_bigquery_source: Required for custom tabular. - The BigQuery table URI representing where the ground truth is located. - Used to provide ground truth for each prediction instance when they are - not part of the batch prediction jobs prediction instance. - classification_type: The type of classification problem, - either ``multiclass`` or ``multilabel``. - class_labels: The list of class names for the - target_field_name, in the same order they appear in the batch - predictions jobs predictions output file. For instance, if the values of - target_field_name could be either ``1`` or ``0``, and the predictions output - contains ["1", "0"] for the prediction_label_column, then the - class_labels input will be ["1", "0"]. If not set, defaults to the - classes found in the prediction_label_column in the batch prediction - jobs predictions file. - target_field_name: The full name path of the features target field - in the predictions file. Formatted to be able to find nested columns, - delimited by ``.``. Alternatively referred to as the ground truth (or - ground_truth_column) field. - model: The Vertex model used for evaluation. Must be located in the same - region as the location argument. It is used to set the default - configurations for AutoML and custom-trained models. - prediction_score_column: The column name of the field - containing batch prediction scores. Formatted to be able to find nested - columns, delimited by ``.``. - prediction_label_column: The column name of the field - containing classes the model is scoring. Formatted to be able to find - nested columns, delimited by ``.``. - slicing_specs: List of - ``google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec``. When - provided, compute metrics for each defined slice. See sample code in - https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component - Below is an example of how to format this input. - - 1: First, create a SlicingSpec. - ``from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice import SliceSpec`` - - ``from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice.SliceSpec import SliceConfig`` + predictions_format: The file format for the batch prediction results. `jsonl`, `csv`, and `bigquery` are the allowed formats, from Vertex Batch Prediction. + predictions_gcs_source: An artifact with its URI pointing toward a GCS directory with prediction or explanation files to be used for this evaluation. For prediction results, the files should be named "prediction.results-*" or "predictions_". For explanation results, the files should be named "explanation.results-*". + predictions_bigquery_source: BigQuery table with prediction or explanation data to be used for this evaluation. For prediction results, the table column should be named "predicted_*". + ground_truth_format: Required for custom tabular and non tabular data. The file format for the ground truth files. `jsonl`, `csv`, and `bigquery` are the allowed formats. + ground_truth_gcs_source: Required for custom tabular and non tabular data. The GCS URIs representing where the ground truth is located. Used to provide ground truth for each prediction instance when they are not part of the batch prediction jobs prediction instance. + ground_truth_bigquery_source: Required for custom tabular. The BigQuery table URI representing where the ground truth is located. Used to provide ground truth for each prediction instance when they are not part of the batch prediction jobs prediction instance. + classification_type: The type of classification problem, either `multiclass` or `multilabel`. + class_labels: The list of class names for the target_field_name, in the same order they appear in the batch predictions jobs predictions output file. For instance, if the values of target_field_name could be either `1` or `0`, and the predictions output contains ["1", "0"] for the prediction_label_column, then the class_labels input will be ["1", "0"]. If not set, defaults to the classes found in the prediction_label_column in the batch prediction jobs predictions file. + target_field_name: The full name path of the features target field in the predictions file. Formatted to be able to find nested columns, delimited by `.`. Alternatively referred to as the ground truth (or ground_truth_column) field. + model: The Vertex model used for evaluation. Must be located in the same region as the location argument. It is used to set the default configurations for AutoML and custom-trained models. + prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + prediction_label_column: The column name of the field containing classes the model is scoring. Formatted to be able to find nested columns, delimited by `.`. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See sample code in https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component Below is an example of how to format this input. - ``slicing_spec = SliceSpec(configs={ 'feature_a': SliceConfig(SliceSpec.Value(string_value='label_a'))})`` - 2: Create a list to store the slicing specs into. - ``slicing_specs = []`` - 3: Format each SlicingSpec into a JSON or Dict. - ``slicing_spec_json = json_format.MessageToJson(slicing_spec)`` - or - ``slicing_spec_dict = json_format.MessageToDict(slicing_spec)`` - 4: Combine each slicing_spec JSON into a list. - ``slicing_specs.append(slicing_spec_json)`` - 5: Finally, pass slicing_specs as an parameter for this component. - ``ModelEvaluationClassificationOp(slicing_specs=slicing_specs)`` - For more details on configuring slices, see - https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice - positive_classes: The list of class - names to create binary classification metrics based on one-vs-rest for - each value of positive_classes provided. - dataflow_service_account: Service account to run - the Dataflow job. If not set, Dataflow will use the default worker - service account. For more details, see - https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account - dataflow_disk_size_gb: The disk size (in GB) of the machine - executing the evaluation run. - dataflow_machine_type: The machine type executing the - evaluation run. - dataflow_workers_num: The number of workers executing the - evaluation run. - dataflow_max_workers_num: The max number of workers - executing the evaluation run. - dataflow_subnetwork: Dataflow's fully qualified subnetwork - name, when empty the default subnetwork will be used. More - details: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow - workers use public IP addresses. - encryption_spec_key_name: Customer-managed encryption key options. - If set, resources created by this pipeline will be encrypted with the - provided encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Flag to choose Beam runner. Valid options are ``DirectRunner`` - and ``Dataflow``. + 1: First, create a SlicingSpec. `from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice import SliceSpec` `from google.cloud.aiplatform_v1.types.ModelEvaluationSlice.Slice.SliceSpec import SliceConfig` `slicing_spec = SliceSpec(configs={ 'feature_a': SliceConfig(SliceSpec.Value(string_value='label_a'))})` + 2: Create a list to store the slicing specs into. `slicing_specs = []` + 3: Format each SlicingSpec into a JSON or Dict. `slicing_spec_json = json_format.MessageToJson(slicing_spec)` or `slicing_spec_dict = json_format.MessageToDict(slicing_spec)` + 4: Combine each slicing_spec JSON into a list. `slicing_specs.append(slicing_spec_json)` + 5: Finally, pass slicing_specs as an parameter for this component. `ModelEvaluationClassificationOp(slicing_specs=slicing_specs)` For more details on configuring slices, see https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice + positive_classes: The list of class names to create binary classification metrics based on one-vs-rest for each value of positive_classes provided. + dataflow_service_account: Service account to run the Dataflow job. If not set, Dataflow will use the default worker service account. For more details, see https://cloud.google.com/dataflow/docs/concepts/security-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. + dataflow_machine_type: The machine type executing the evaluation run. + dataflow_workers_num: The number of workers executing the evaluation run. + dataflow_max_workers_num: The max number of workers executing the evaluation run. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + force_runner_mode: Flag to choose Beam runner. Valid options are `DirectRunner` and `Dataflow`. + project: Project to run evaluation container. Defaults to the project in which the PipelineJob is run. Returns: - evaluation_metrics: - ``google.ClassificationMetrics`` representing the classification - evaluation metrics in GCS. - gcp_resources: Serialized gcp_resources proto tracking the Dataflow - job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + evaluation_metrics: `google.ClassificationMetrics` representing the classification evaluation metrics in GCS. + gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return dsl.ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/error_analysis_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/error_analysis_pipeline.py index 881b3d4ef3..43f09e4916 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/error_analysis_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/error_analysis_pipeline.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components._implementation.model import GetVertexModelOp from google_cloud_pipeline_components._implementation.model_evaluation import ErrorAnalysisAnnotationOp from google_cloud_pipeline_components._implementation.model_evaluation import EvaluatedAnnotationOp @@ -22,13 +25,11 @@ from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from google_cloud_pipeline_components.v1.dataset import GetVertexDatasetOp from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp -import kfp from kfp import dsl -@kfp.dsl.pipeline(name='vision-model-error-analysis-pipeline') +@dsl.pipeline(name='automl-vision-error-analysis-pipeline') def vision_model_error_analysis_pipeline( # pylint: disable=dangerous-default-value - project: str, location: str, model_name: str, batch_predict_gcs_destination_output_uri: str, @@ -36,8 +37,8 @@ def vision_model_error_analysis_pipeline( # pylint: disable=dangerous-default-v test_dataset_annotation_set_name: str = '', training_dataset_resource_name: str = '', training_dataset_annotation_set_name: str = '', - test_dataset_storage_source_uris: list = [], - training_dataset_storage_source_uris: list = [], + test_dataset_storage_source_uris: List[str] = [], + training_dataset_storage_source_uris: List[str] = [], batch_predict_instances_format: str = 'jsonl', batch_predict_predictions_format: str = 'jsonl', batch_predict_machine_type: str = 'n1-standard-32', @@ -52,8 +53,11 @@ def vision_model_error_analysis_pipeline( # pylint: disable=dangerous-default-v dataflow_subnetwork: str = '', dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-vision-error-analysis-pipeline-{{$.pipeline_job_uuid}}', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): + # fmt: off """The evaluation vision error analysis pipeline. This pipeline can help you to continuously discover dataset example errors @@ -62,104 +66,36 @@ def vision_model_error_analysis_pipeline( # pylint: disable=dangerous-default-v including Dataflow and BatchPrediction. Args: - project: The GCP project that runs the pipeline components. location: The GCP region that runs the pipeline components. - model_name: The Vertex model resource name to be imported and used for batch - prediction, in the format of - projects/{project}/locations/{location}/models/{model} or - projects/{project}/locations/{location}/models/{model}@{model_version_id - or model_version_alias} - batch_predict_gcs_destination_output_uri: The Google Cloud Storage location - of the directory where the output is to be written to. In the given - directory a new directory is created. Its name is - ``prediction--``, where timestamp is - in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files - ``predictions_0001.``, ``predictions_0002.``, ..., - ``predictions_N.`` are created where ```` depends on - chosen ``predictions_format``, and N may equal 0001 and depends on the - total number of successfully predicted instances. If the Model has both - ``instance`` and ``prediction`` schemata defined then each such file - contains predictions as per the ``predictions_format``. If prediction for - any instance failed (partially or completely), then an additional - ``errors_0001.``, ``errors_0002.``,..., - ``errors_N.`` files are created (N depends on total number of - failed predictions). These files contain the failed instances, as per - their schema, followed by an additional ``error`` field which as value has - ``google.rpc.Status`` containing only ``code`` and ``message`` fields. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - test_dataset_resource_name: A Vertex dataset resource name of the test - dataset. If ``test_dataset_storage_source_uris`` is also provided, this - argument will override the GCS source. - test_dataset_annotation_set_name: A string of the annotation_set resource - name containing the ground truth of the test datset used for evaluation. - training_dataset_resource_name: A Vertex dataset resource name of the - training dataset. If ``training_dataset_storage_source_uris`` is also - provided, this argument will override the GCS source. - training_dataset_annotation_set_name: A string of the annotation_set - resource name containing the ground truth of the test datset used for - feature extraction. - test_dataset_storage_source_uris: Google Cloud Storage URI(-s) to unmanaged - test datasets.``jsonl`` is currently the only allowed format. If - ``test_dataset`` is also provided, this field will be overriden by the - provided Vertex Dataset. - training_dataset_storage_source_uris: Google Cloud Storage URI(-s) to - unmanaged test datasets.``jsonl`` is currently the only allowed format. If - ``training_dataset`` is also provided, this field will be overriden by the - provided Vertex Dataset. - batch_predict_instances_format: The format in which instances are given, - must be one of the Model's supportedInputStorageFormats. For more details - about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_predictions_format: The format in which Vertex AI gives the - predictions. Must be one of the Model's supportedOutputStorageFormats. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_machine_type: The type of machine for running batch prediction - on dedicated resources. If the Model supports DEDICATED_RESOURCES this - config may be provided (and the job will use these resources). If the - Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. - For more details about the BatchDedicatedResources, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. - For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_starting_replica_count: The number of machine replicas used at - the start of the batch operation. If not set, Vertex AI decides starting - number, not greater than ``max_replica_count``. Only used if - ``machine_type`` is set. - batch_predict_max_replica_count: The maximum number of machine replicas the - batch operation may be scaled to. Only used if ``machine_type`` is set. - batch_predict_accelerator_type: The type of accelerator(s) that may be - attached to the machine as per ``batch_predict_accelerator_count``. Only - used if ``batch_predict_machine_type`` is set. For more details about the - machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_accelerator_count: The number of accelerators to attach to the - ``batch_predict_machine_type``. Only used if - ``batch_predict_machine_type`` is set. + model_name: The Vertex model resource name to be imported and used for batch prediction, in the format of `projects/{project}/locations/{location}/models/{model}` or `projects/{project}/locations/{location}/models/{model}@{model_version_id or model_version_alias}` + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + test_dataset_resource_name: A Vertex dataset resource name of the test dataset. If `test_dataset_storage_source_uris` is also provided, this argument will override the GCS source. + test_dataset_annotation_set_name: A string of the annotation_set resource name containing the ground truth of the test datset used for evaluation. + training_dataset_resource_name: A Vertex dataset resource name of the training dataset. If `training_dataset_storage_source_uris` is also provided, this argument will override the GCS source. + training_dataset_annotation_set_name: A string of the annotation_set resource name containing the ground truth of the test datset used for feature extraction. + test_dataset_storage_source_uris: Google Cloud Storage URI(-s) to unmanaged test datasets.`jsonl` is currently the only allowed format. If `test_dataset` is also provided, this field will be overridden by the provided Vertex Dataset. + training_dataset_storage_source_uris: Google Cloud Storage URI(-s) to unmanaged test datasets.`jsonl` is currently the only allowed format. If `training_dataset` is also provided, this field will be overridden by the provided Vertex Dataset. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. dataflow_machine_type: The Dataflow machine type for evaluation components. - dataflow_max_num_workers: The max number of Dataflow workers for evaluation - components. - dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation - components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. dataflow_service_account: Custom service account to run Dataflow jobs. - dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty - the default subnetwork will be used. Example: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow workers use public IP - addresses. - encryption_spec_key_name: Customer-managed encryption key options. If set, - resources created by this pipeline will be encrypted with the provided - encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Indicate the runner mode to use forcely. Valid options - are ``Dataflow`` and ``DirectRunner``. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the project in which the PipelineJob is run. """ - evaluation_display_name = 'vision-model-error-analysis-pipeline' + # fmt: on - with kfp.dsl.Condition( + with dsl.Condition( ( test_dataset_resource_name != '' and training_dataset_resource_name != '' @@ -272,9 +208,7 @@ def vision_model_error_analysis_pipeline( # pylint: disable=dangerous-default-v dataset_paths=dataset_preprocessor_task.outputs[ 'batch_prediction_storage_source' ], - display_name=( - f'{evaluation_display_name}-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}' - ), + display_name=evaluation_display_name, ) ModelImportEvaluatedAnnotationOp( model=get_model_task.outputs['model'], @@ -289,7 +223,7 @@ def vision_model_error_analysis_pipeline( # pylint: disable=dangerous-default-v ], ) - with kfp.dsl.Condition( + with dsl.Condition( ( ( test_dataset_resource_name == '' @@ -392,9 +326,7 @@ def vision_model_error_analysis_pipeline( # pylint: disable=dangerous-default-v dataset_paths=dataset_preprocessor_task.outputs[ 'batch_prediction_storage_source' ], - display_name=( - f'{evaluation_display_name}-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}' - ), + display_name=evaluation_display_name, ) ModelImportEvaluatedAnnotationOp( model=get_model_task.outputs['model'], diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluated_annotation_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluated_annotation_pipeline.py index fdd02830f1..cfb08646d8 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluated_annotation_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluated_annotation_pipeline.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components._implementation.model import GetVertexModelOp from google_cloud_pipeline_components._implementation.model_evaluation import EvaluatedAnnotationOp from google_cloud_pipeline_components._implementation.model_evaluation import EvaluationDatasetPreprocessorOp as DatasetPreprocessorOp @@ -20,18 +23,17 @@ from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from google_cloud_pipeline_components.v1.dataset import GetVertexDatasetOp from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp -import kfp +from kfp import dsl -@kfp.dsl.pipeline(name='evaluated-annotation-pipeline') -def evaluated_annotation_pipeline( - project: str, +@dsl.pipeline(name='automl-vision-evaluated-annotation-pipeline') +def evaluated_annotation_pipeline( # pylint: disable=dangerous-default-value location: str, model_name: str, batch_predict_gcs_destination_output_uri: str, test_dataset_resource_name: str = '', test_dataset_annotation_set_name: str = '', - test_dataset_storage_source_uris: list = [], + test_dataset_storage_source_uris: List[str] = [], batch_predict_instances_format: str = 'jsonl', batch_predict_predictions_format: str = 'jsonl', batch_predict_machine_type: str = 'n1-standard-32', @@ -46,95 +48,40 @@ def evaluated_annotation_pipeline( dataflow_subnetwork: str = '', dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-vision-evaluated-annotation-pipeline-{{$.pipeline_job_uuid}}', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): + # fmt: off """The evaluation evaluated annotation pipeline. Args: - project: The GCP project that runs the pipeline components. location: The GCP region that runs the pipeline components. - model_name: The Vertex model resource name to be imported and used for batch - prediction. - batch_predict_gcs_destination_output_uri: The Google Cloud Storage location - of the directory where the output is to be written to. In the given - directory a new directory is created. Its name is - ``prediction--``, where timestamp is - in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files - ``predictions_0001.``, ``predictions_0002.``, ..., - ``predictions_N.`` are created where ```` depends on - chosen ``predictions_format``, and N may equal 0001 and depends on the - total number of successfully predicted instances. If the Model has both - ``instance`` and ``prediction`` schemata defined then each such file - contains predictions as per the ``predictions_format``. If prediction for - any instance failed (partially or completely), then an additional - ``errors_0001.``, ``errors_0002.``,..., - ``errors_N.`` files are created (N depends on total number of - failed predictions). These files contain the failed instances, as per - their schema, followed by an additional ``error`` field which as value has - ``google.rpc.Status`` containing only ``code`` and ``message`` fields. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - test_dataset_resource_name: A Vertex dataset resource name of the test - dataset. If ``test_dataset_storage_source_uris`` is also provided, this - argument will override the GCS source. - test_dataset_annotation_set_name: A string of the annotation_set name - containing the ground truth of the test datset used for evaluation. - test_dataset_storage_source_uris: Google Cloud Storage URI(-s) to unmanaged - test datasets.``jsonl`` is currently the only allowed format. If - ``test_dataset`` is also provided, this field will be overriden by the - provided Vertex Dataset. - batch_predict_instances_format: The format in which instances are given, - must be one of the Model's supportedInputStorageFormats. For more details - about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_predictions_format: The format in which Vertex AI gives the - predictions. Must be one of the Model's supportedOutputStorageFormats. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_machine_type: The type of machine for running batch prediction - on dedicated resources. If the Model supports DEDICATED_RESOURCES this - config may be provided (and the job will use these resources). If the - Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. - For more details about the BatchDedicatedResources, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. - For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_starting_replica_count: The number of machine replicas used at - the start of the batch operation. If not set, Vertex AI decides starting - number, not greater than ``max_replica_count``. Only used if - ``machine_type`` is set. - batch_predict_max_replica_count: The maximum number of machine replicas the - batch operation may be scaled to. Only used if ``machine_type`` is set. - batch_predict_accelerator_type: The type of accelerator(s) that may be - attached to the machine as per ``batch_predict_accelerator_count``. Only - used if ``batch_predict_machine_type`` is set. For more details about the - machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_accelerator_count: The number of accelerators to attach to the - ``batch_predict_machine_type``. Only used if - ``batch_predict_machine_type`` is set. + model_name: The Vertex model resource name to be imported and used for batch prediction, in the format of `projects/{project}/locations/{location}/models/{model}` or `projects/{project}/locations/{location}/models/{model}@{model_version_id or model_version_alias}` + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + test_dataset_resource_name: A Vertex dataset resource name of the test dataset. If `test_dataset_storage_source_uris` is also provided, this argument will override the GCS source. + test_dataset_annotation_set_name: A string of the annotation_set name containing the ground truth of the test datset used for evaluation. + test_dataset_storage_source_uris: Google Cloud Storage URI(-s) to unmanaged test datasets.`jsonl` is currently the only allowed format. If `test_dataset` is also provided, this field will be overridden by the provided Vertex Dataset. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. dataflow_machine_type: The Dataflow machine type for evaluation components. - dataflow_max_num_workers: The max number of Dataflow workers for evaluation - components. - dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation - components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. dataflow_service_account: Custom service account to run Dataflow jobs. - dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty - the default subnetwork will be used. Example: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow workers use public IP - addresses. - encryption_spec_key_name: Customer-managed encryption key options. If set, - resources created by this pipeline will be encrypted with the provided - encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Indicate the runner mode to use forcely. Valid options - are ``Dataflow`` and ``DirectRunner``. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the project in which the PipelineJob is run. """ - evaluation_display_name = 'evaluated-annotation-pipeline' - get_model_task = GetVertexModelOp(model_name=model_name) + # fmt: off + get_test_dataset_task = GetVertexDatasetOp( dataset_resource_name=test_dataset_resource_name ) @@ -145,7 +92,7 @@ def evaluated_annotation_pipeline( test_dataset_annotation_set_name=test_dataset_annotation_set_name, test_dataset_storage_source_uris=test_dataset_storage_source_uris, ) - + get_model_task = GetVertexModelOp(model_name=model_name) batch_predict_task = ModelBatchPredictOp( project=project, location=location, @@ -203,7 +150,6 @@ def evaluated_annotation_pipeline( dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, ) - model_evaluation_importer_task = ModelImportEvaluationOp( classification_metrics=eval_task.outputs['evaluation_metrics'], model=get_model_task.outputs['model'], @@ -213,7 +159,6 @@ def evaluated_annotation_pipeline( ], display_name=evaluation_display_name, ) - ModelImportEvaluatedAnnotationOp( model=get_model_task.outputs['model'], evaluated_annotation_output_uri=evaluated_annotation_task.outputs[ diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_feature_attribution_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_feature_attribution_pipeline.py index b2d1e7d3e4..e8db28b4a3 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_feature_attribution_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_feature_attribution_pipeline.py @@ -12,39 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, Dict, List, NamedTuple + +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components._implementation.model import GetVertexModelOp -from google_cloud_pipeline_components._implementation.model_evaluation import EvaluationDataSamplerOp +from google_cloud_pipeline_components._implementation.model_evaluation import FeatureAttributionGraphComponentOp from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp -from google_cloud_pipeline_components.preview.model_evaluation import ModelEvaluationFeatureAttributionOp +from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics +from google_cloud_pipeline_components.types.artifact_types import RegressionMetrics from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp -from google_cloud_pipeline_components.v1.model_evaluation.forecasting_component import model_evaluation_forecasting as ModelEvaluationForecastingOp from google_cloud_pipeline_components.v1.model_evaluation.regression_component import model_evaluation_regression as ModelEvaluationRegressionOp import kfp -@kfp.dsl.pipeline(name='evaluation-automl-tabular-feature-attribution-pipeline') -def evaluation_automl_tabular_feature_attribution_pipeline( # pylint: disable=dangerous-default-value - project: str, +@kfp.dsl.pipeline( + name='evaluation-automl-tabular-feature-attribution-classification-pipeline' +) +def evaluation_automl_tabular_feature_attribution_classification_pipeline( # pylint: disable=dangerous-default-value location: str, - prediction_type: str, model_name: str, target_field_name: str, batch_predict_instances_format: str, batch_predict_gcs_destination_output_uri: str, - batch_predict_gcs_source_uris: list = [], # pylint: disable=g-bare-generic + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic batch_predict_bigquery_source_uri: str = '', batch_predict_predictions_format: str = 'jsonl', batch_predict_bigquery_destination_output_uri: str = '', batch_predict_machine_type: str = 'n1-standard-16', batch_predict_starting_replica_count: int = 5, batch_predict_max_replica_count: int = 10, - batch_predict_explanation_metadata: dict = {}, # pylint: disable=g-bare-generic - batch_predict_explanation_parameters: dict = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_metadata: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: Dict[str, Any] = {}, # pylint: disable=g-bare-generic batch_predict_explanation_data_sample_size: int = 10000, batch_predict_accelerator_type: str = '', batch_predict_accelerator_count: int = 0, - slicing_specs: list = [], # pylint: disable=g-bare-generic + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic + evaluation_display_name: str = 'evaluation-automl-tabular-feature-attribution-pipeline-{{$.pipeline_job_uuid}}', dataflow_machine_type: str = 'n1-standard-4', dataflow_max_num_workers: int = 5, dataflow_disk_size_gb: int = 50, @@ -53,142 +57,58 @@ def evaluation_automl_tabular_feature_attribution_pipeline( # pylint: disable=d dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +) -> NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, ): - """The evaluation AutoML tabular pipeline with feature attribution. + # fmt: off + """The evaluation AutoML tabular pipeline with feature attribution for classification models. - This pipeline guarantees support for AutoML Tabular models that contain a - valid explanation_spec. This pipeline does not include the data_splitter - component, which is needed for many tabular custom models. + This pipeline guarantees support for AutoML Tabular models that contain a valid explanation_spec. This pipeline does not include the target_field_data_remover component, which is needed for many tabular custom models. Args: - project: The GCP project that runs the pipeline components. location: The GCP region that runs the pipeline components. - prediction_type: The type of prediction the model is to produce. - "classification", "regression", or "forecasting". - model_name: The Vertex model resource name to be imported and used for batch - prediction. - target_field_name: The target field's name. Formatted to be able to find - nested columns, delimited by ``.``. Prefixed with 'instance.' on the - component for Vertex Batch Prediction. - batch_predict_instances_format: The format in which instances are given, - must be one of the Model's supportedInputStorageFormats. For more details - about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_gcs_destination_output_uri: The Google Cloud Storage location - of the directory where the output is to be written to. In the given - directory a new directory is created. Its name is - ``prediction--``, where timestamp is - in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files - ``predictions_0001.``, ``predictions_0002.``, ..., - ``predictions_N.`` are created where ```` depends on - chosen ``predictions_format``, and N may equal 0001 and depends on the - total number of successfully predicted instances. If the Model has both - ``instance`` and ``prediction`` schemata defined then each such file - contains predictions as per the ``predictions_format``. If prediction for - any instance failed (partially or completely), then an additional - ``errors_0001.``, ``errors_0002.``,..., - ``errors_N.`` files are created (N depends on total number of - failed predictions). These files contain the failed instances, as per - their schema, followed by an additional ``error`` field which as value has - ``google.rpc.Status`` containing only ``code`` and ``message`` fields. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your - instances to run batch prediction on. May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For - more details about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to - run batch prediction on. May contain wildcards. For more details about - this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_predictions_format: The format in which Vertex AI gives the - predictions. Must be one of the Model's supportedOutputStorageFormats. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_bigquery_destination_output_uri: The BigQuery project location - where the output is to be written to. In the given project a new dataset - is created with name ``prediction__`` - where is made BigQuery-dataset-name compatible (for example, most special - characters become underscores), and timestamp is in - YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two - tables will be created, ``predictions``, and ``errors``. If the Model has - both ``instance`` and ``prediction`` schemata defined then the tables have - columns as follows: The ``predictions`` table contains instances for which - the prediction succeeded, it has columns as per a concatenation of the - Model's instance and prediction schemata. The ``errors`` table contains - rows for which the prediction has failed, it has instance columns, as per - the instance schema, followed by a single "errors" column, which as values - has ````google.rpc.Status`` ``__ represented as a STRUCT, and - containing only ``code`` and ``message``. For more details about this - output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_machine_type: The type of machine for running batch prediction - on dedicated resources. If the Model supports DEDICATED_RESOURCES this - config may be provided (and the job will use these resources). If the - Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. - For more details about the BatchDedicatedResources, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. - For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_starting_replica_count: The number of machine replicas used at - the start of the batch operation. If not set, Vertex AI decides starting - number, not greater than ``max_replica_count``. Only used if - ``machine_type`` is set. - batch_predict_max_replica_count: The maximum number of machine replicas the - batch operation may be scaled to. Only used if ``machine_type`` is set. - batch_predict_explanation_metadata: Explanation metadata configuration for - this BatchPredictionJob. Can be specified only if ``generate_explanation`` - is set to ``True``. This value overrides the value of - ``Model.explanation_metadata``. All fields of ``explanation_metadata`` are - optional in the request. If a field of the ``explanation_metadata`` object - is not populated, the corresponding field of the - ``Model.explanation_metadata`` object is inherited. For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. - batch_predict_explanation_parameters: Parameters to configure explaining for - Model's predictions. Can be specified only if ``generate_explanation`` is - set to ``True``. This value overrides the value of - ``Model.explanation_parameters``. All fields of ``explanation_parameters`` - are optional in the request. If a field of the ``explanation_parameters`` - object is not populated, the corresponding field of the - ``Model.explanation_parameters`` object is inherited. For more details, - see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. - batch_predict_explanation_data_sample_size: Desired size to downsample the - input dataset that will then be used for batch explanation. - batch_predict_accelerator_type: The type of accelerator(s) that may be - attached to the machine as per ``batch_predict_accelerator_count``. Only - used if ``batch_predict_machine_type`` is set. For more details about the - machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_accelerator_count: The number of accelerators to attach to the - ``batch_predict_machine_type``. Only used if - ``batch_predict_machine_type`` is set. + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See sample code in https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component For more details on configuring slices, see https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice. dataflow_machine_type: The Dataflow machine type for evaluation components. - dataflow_max_num_workers: The max number of Dataflow workers for evaluation - components. - dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation - components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. dataflow_service_account: Custom service account to run Dataflow jobs. - dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty - the default subnetwork will be used. Example: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow workers use public IP - addresses. - encryption_spec_key_name: Customer-managed encryption key options. If set, - resources created by this pipeline will be encrypted with the provided - encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Indicate the runner mode to use forcely. Valid options - are ``Dataflow`` and ``DirectRunner``. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the project in which the PipelineJob is run. + + Returns: + A google.ClassificationMetrics artifact. """ - evaluation_display_name = ( - 'evaluation-automl-tabular-feature-attribution-pipeline' + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, ) + get_model_task = GetVertexModelOp(model_name=model_name) # Run Batch Prediction. @@ -211,32 +131,173 @@ def evaluation_automl_tabular_feature_attribution_pipeline( # pylint: disable=d accelerator_count=batch_predict_accelerator_count, ) - # Run the Batch Explain process (sampler -> batch explanation). - data_sampler_task = EvaluationDataSamplerOp( + # Run feature attribution steps. + feature_attribution_graph = FeatureAttributionGraphComponentOp( project=project, location=location, - gcs_source_uris=batch_predict_gcs_source_uris, - bigquery_source_uri=batch_predict_bigquery_source_uri, - instances_format=batch_predict_instances_format, - sample_size=batch_predict_explanation_data_sample_size, + prediction_type='classification', + vertex_model=get_model_task.outputs['model'], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + dataflow_machine_type=dataflow_machine_type, + dataflow_max_num_workers=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, force_runner_mode=force_runner_mode, ) - batch_explain_task = ModelBatchPredictOp( + + # Run evaluation for a classification model. + eval_task = ModelEvaluationClassificationOp( project=project, location=location, - model=get_model_task.outputs['model'], - job_display_name='model-registry-batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', - gcs_source_uris=data_sampler_task.outputs['gcs_output_directory'], - bigquery_source_input_uri=data_sampler_task.outputs[ + target_field_name=target_field_name, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ 'bigquery_output_table' ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + slicing_specs=slicing_specs, + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + classification_metrics=eval_task.outputs['evaluation_metrics'], + feature_attributions=feature_attribution_graph.outputs[ + 'feature_attributions' + ], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline( + name='evaluation-automl-tabular-feature-attribution-regression-pipeline' +) +def evaluation_automl_tabular_feature_attribution_regression_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_explanation_metadata: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_data_sample_size: int = 10000, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-tabular-feature-attribution-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', +) -> NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, +): + # fmt: off + """The evaluation AutoML tabular pipeline with feature attribution for regression models. + + This pipeline guarantees support for AutoML Tabular models that contain a valid explanation_spec. This pipeline does not include the target_field_data_remover component, which is needed for many tabular custom models. + + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + + Returns: A google.RegressionMetrics artifact. + """ + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, + ) + + get_model_task = GetVertexModelOp(model_name=model_name) + + # Run Batch Prediction. + batch_predict_task = ModelBatchPredictOp( + project=project, + location=location, + model=get_model_task.outputs['model'], + job_display_name='model-registry-batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=batch_predict_gcs_source_uris, + bigquery_source_input_uri=batch_predict_bigquery_source_uri, instances_format=batch_predict_instances_format, predictions_format=batch_predict_predictions_format, gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, - generate_explanation=True, - explanation_parameters=batch_predict_explanation_parameters, - explanation_metadata=batch_predict_explanation_metadata, machine_type=batch_predict_machine_type, starting_replica_count=batch_predict_starting_replica_count, max_replica_count=batch_predict_max_replica_count, @@ -245,169 +306,211 @@ def evaluation_automl_tabular_feature_attribution_pipeline( # pylint: disable=d accelerator_count=batch_predict_accelerator_count, ) - # Run evaluation based on prediction type and feature attribution component. - # After, import the model evaluations to the Vertex model. + # Run feature attribution steps. + feature_attribution_graph = FeatureAttributionGraphComponentOp( + project=project, + location=location, + prediction_type='regression', + vertex_model=get_model_task.outputs['model'], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + dataflow_machine_type=dataflow_machine_type, + dataflow_max_num_workers=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + ) + + # Run evaluation for a regression model. + eval_task = ModelEvaluationRegressionOp( + project=project, + location=location, + target_field_name=target_field_name, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + regression_metrics=eval_task.outputs['evaluation_metrics'], + feature_attributions=feature_attribution_graph.outputs[ + 'feature_attributions' + ], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline(name='evaluation-automl-tabular-feature-attribution-pipeline') +def evaluation_automl_tabular_feature_attribution_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + prediction_type: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_explanation_metadata: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_data_sample_size: int = 10000, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-tabular-feature-attribution-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', +): + # fmt: off + """The evaluation AutoML tabular pipeline with feature attribution. + + This pipeline guarantees support for AutoML Tabular classification and + regression models that contain a valid explanation_spec. This pipeline does + not include the target_field_data_remover component, which is needed for many + tabular custom models. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See sample code in https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component For more details on configuring slices, see https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + """ + # fmt: on with kfp.dsl.Condition( prediction_type == 'classification', name='classification' ): - eval_task = ModelEvaluationClassificationOp( + evaluation_automl_tabular_feature_attribution_classification_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, slicing_specs=slicing_specs, - ) - feature_attribution_task = ModelEvaluationFeatureAttributionOp( - project=project, - location=location, - problem_type=prediction_type, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_explain_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_explain_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - ) - ModelImportEvaluationOp( - classification_metrics=eval_task.outputs['evaluation_metrics'], - feature_attributions=feature_attribution_task.outputs[ - 'feature_attributions' - ], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, - ) - - with kfp.dsl.Condition(prediction_type == 'forecasting', name='forecasting'): - eval_task = ModelEvaluationForecastingOp( - project=project, - location=location, - target_field_name=target_field_name, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - feature_attribution_task = ModelEvaluationFeatureAttributionOp( - project=project, - location=location, - problem_type=prediction_type, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_explain_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_explain_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - ) - ModelImportEvaluationOp( - forecasting_metrics=eval_task.outputs['evaluation_metrics'], - feature_attributions=feature_attribution_task.outputs[ - 'feature_attributions' - ], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) with kfp.dsl.Condition(prediction_type == 'regression', name='regression'): - eval_task = ModelEvaluationRegressionOp( + evaluation_automl_tabular_feature_attribution_regression_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - feature_attribution_task = ModelEvaluationFeatureAttributionOp( - project=project, - location=location, - problem_type=prediction_type, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_explain_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_explain_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - ) - ModelImportEvaluationOp( - regression_metrics=eval_task.outputs['evaluation_metrics'], - feature_attributions=feature_attribution_task.outputs[ - 'feature_attributions' - ], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_pipeline.py index 0d2b7196b2..fa49509dde 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_tabular_pipeline.py @@ -12,26 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import NamedTuple +from typing import Any, List, NamedTuple + +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components._implementation.model import GetVertexModelOp from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp +from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics +from google_cloud_pipeline_components.types.artifact_types import RegressionMetrics from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp -from google_cloud_pipeline_components.v1.model_evaluation.forecasting_component import model_evaluation_forecasting as ModelEvaluationForecastingOp from google_cloud_pipeline_components.v1.model_evaluation.regression_component import model_evaluation_regression as ModelEvaluationRegressionOp import kfp -@kfp.dsl.pipeline(name='evaluation-automl-tabular-pipeline') -def evaluation_automl_tabular_pipeline( # pylint: disable=dangerous-default-value - project: str, +@kfp.dsl.pipeline(name='evaluation-automl-tabular-classification-pipeline') +def evaluation_automl_tabular_classification_pipeline( # pylint: disable=dangerous-default-value location: str, - prediction_type: str, model_name: str, target_field_name: str, batch_predict_instances_format: str, batch_predict_gcs_destination_output_uri: str, - batch_predict_gcs_source_uris: list = [], # pylint: disable=g-bare-generic + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic batch_predict_bigquery_source_uri: str = '', batch_predict_predictions_format: str = 'jsonl', batch_predict_bigquery_destination_output_uri: str = '', @@ -40,7 +41,8 @@ def evaluation_automl_tabular_pipeline( # pylint: disable=dangerous-default-val batch_predict_max_replica_count: int = 10, batch_predict_accelerator_type: str = '', batch_predict_accelerator_count: int = 0, - slicing_specs: list = [], # pylint: disable=g-bare-generic + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic + evaluation_display_name: str = 'evaluation-automl-tabular-pipeline-{{$.pipeline_job_uuid}}', dataflow_machine_type: str = 'n1-standard-4', dataflow_max_num_workers: int = 5, dataflow_disk_size_gb: int = 50, @@ -49,123 +51,197 @@ def evaluation_automl_tabular_pipeline( # pylint: disable=dangerous-default-val dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +) -> NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, ): - """The evaluation AutoML tabular pipeline with no feature attribution. + # fmt: off + """The evaluation AutoML tabular pipeline with no feature attribution for. + + classification models. This pipeline guarantees support for AutoML Tabular models. This pipeline does not include the target_field_data_remover component, which is needed for many tabular custom models. + Args: + location: The GCP region that runs the pipeline components. + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See sample code in https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component For more details on configuring slices, see https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the project in which the PipelineJob is run. + + Returns: + A google.ClassificationMetrics artifact and imported evaluation_resource_name. + """ + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, + ) + + # Get the Vertex AI Model. + get_model_task = GetVertexModelOp(model_name=model_name) + + # Run Vertex AI Batch Prediction. + batch_predict_task = ModelBatchPredictOp( + project=project, + location=location, + model=get_model_task.outputs['model'], + job_display_name='evaluation-batch-predict-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=batch_predict_gcs_source_uris, + bigquery_source_input_uri=batch_predict_bigquery_source_uri, + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, + bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + machine_type=batch_predict_machine_type, + starting_replica_count=batch_predict_starting_replica_count, + max_replica_count=batch_predict_max_replica_count, + encryption_spec_key_name=encryption_spec_key_name, + accelerator_type=batch_predict_accelerator_type, + accelerator_count=batch_predict_accelerator_count, + ) + + # Run evaluation for a classification model. + eval_task = ModelEvaluationClassificationOp( + project=project, + location=location, + target_field_name=target_field_name, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + slicing_specs=slicing_specs, + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + classification_metrics=eval_task.outputs['evaluation_metrics'], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline(name='evaluation-automl-tabular-regression-pipeline') +def evaluation_automl_tabular_regression_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-tabular-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', +) -> NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, +): + # fmt: off + """The evaluation AutoML tabular pipeline with no feature attribution for regression models. + + This pipeline guarantees support for AutoML Tabular models. This pipeline does not include the target_field_data_remover component, which is needed for many tabular custom models. + Args: project: The GCP project that runs the pipeline components. location: The GCP region that runs the pipeline components. - prediction_type: The type of prediction the model is to produce. - "classification", "regression", or "forecasting". - model_name: The Vertex model resource name to be imported and used for batch - prediction. - target_field_name: The target field's name. Formatted to be able to find - nested columns, delimited by ``.``. Prefixed with 'instance.' on the - component for Vertex Batch Prediction. - batch_predict_instances_format: The format in which instances are given, - must be one of the Model's supportedInputStorageFormats. For more details - about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_gcs_destination_output_uri: The Google Cloud Storage location - of the directory where the output is to be written to. In the given - directory a new directory is created. Its name is - ``prediction--``, where timestamp is - in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files - ``predictions_0001.``, ``predictions_0002.``, ..., - ``predictions_N.`` are created where ```` depends on - chosen ``predictions_format``, and N may equal 0001 and depends on the - total number of successfully predicted instances. If the Model has both - ``instance`` and ``prediction`` schemata defined then each such file - contains predictions as per the ``predictions_format``. If prediction for - any instance failed (partially or completely), then an additional - ``errors_0001.``, ``errors_0002.``,..., - ``errors_N.`` files are created (N depends on total number of - failed predictions). These files contain the failed instances, as per - their schema, followed by an additional ``error`` field which as value has - ``google.rpc.Status`` containing only ``code`` and ``message`` fields. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your - instances to run batch prediction on. May contain wildcards. For more - information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For - more details about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to - run batch prediction on. May contain wildcards. For more details about - this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_predictions_format: The format in which Vertex AI gives the - predictions. Must be one of the Model's supportedOutputStorageFormats. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_bigquery_destination_output_uri: The BigQuery project location - where the output is to be written to. In the given project a new dataset - is created with name ``prediction__`` - where is made BigQuery-dataset-name compatible (for example, most special - characters become underscores), and timestamp is in - YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two - tables will be created, ``predictions``, and ``errors``. If the Model has - both ``instance`` and ``prediction`` schemata defined then the tables have - columns as follows: The ``predictions`` table contains instances for which - the prediction succeeded, it has columns as per a concatenation of the - Model's instance and prediction schemata. The ``errors`` table contains - rows for which the prediction has failed, it has instance columns, as per - the instance schema, followed by a single "errors" column, which as values - has ````google.rpc.Status`` ``__ represented as a STRUCT, and - containing only ``code`` and ``message``. For more details about this - output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_machine_type: The type of machine for running batch prediction - on dedicated resources. If the Model supports DEDICATED_RESOURCES this - config may be provided (and the job will use these resources). If the - Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. - For more details about the BatchDedicatedResources, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. - For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_starting_replica_count: The number of machine replicas used at - the start of the batch operation. If not set, Vertex AI decides starting - number, not greater than ``max_replica_count``. Only used if - ``machine_type`` is set. - batch_predict_max_replica_count: The maximum number of machine replicas the - batch operation may be scaled to. Only used if ``machine_type`` is set. - batch_predict_accelerator_type: The type of accelerator(s) that may be - attached to the machine as per ``batch_predict_accelerator_count``. Only - used if ``batch_predict_machine_type`` is set. For more details about the - machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_accelerator_count: The number of accelerators to attach to the - ``batch_predict_machine_type``. Only used if - ``batch_predict_machine_type`` is set. + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. dataflow_machine_type: The Dataflow machine type for evaluation components. - dataflow_max_num_workers: The max number of Dataflow workers for evaluation - components. - dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation - components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. dataflow_service_account: Custom service account to run Dataflow jobs. - dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty - the default subnetwork will be used. Example: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow workers use public IP - addresses. - encryption_spec_key_name: Customer-managed encryption key options. If set, - resources created by this pipeline will be encrypted with the provided - encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Indicate the runner mode to use forcely. Valid options - are ``Dataflow`` and ``DirectRunner``. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + + Returns: + A google.RegressionMetrics artifact and imported evaluation_resource_name. """ - evaluation_display_name = 'evaluation-automl-tabular-pipeline' + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, + ) + + # Get the Vertex AI Model. get_model_task = GetVertexModelOp(model_name=model_name) - # Run Batch Prediction. + # Run Vertex AI Batch Prediction. batch_predict_task = ModelBatchPredictOp( project=project, location=location, @@ -185,100 +261,163 @@ def evaluation_automl_tabular_pipeline( # pylint: disable=dangerous-default-val accelerator_count=batch_predict_accelerator_count, ) - # Run evaluation based on prediction type. - # After, import the model evaluations to the Vertex model. + # Run evaluation for a regression model. + eval_task = ModelEvaluationRegressionOp( + project=project, + location=location, + target_field_name=target_field_name, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + regression_metrics=eval_task.outputs['evaluation_metrics'], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline(name='evaluation-automl-tabular-pipeline') +def evaluation_automl_tabular_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + prediction_type: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-tabular-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', +): + # fmt: off + """The evaluation AutoML tabular pipeline with no feature attribution. + + This pipeline guarantees support for AutoML Tabular classification and regression models. This pipeline does not include the target_field_data_remover component, which is needed for many tabular custom models and AutoML Tabular Forecasting. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances to run batch prediction on. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See sample code in https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component For more details on configuring slices, see https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + """ + # fmt: on with kfp.dsl.Condition( prediction_type == 'classification', name='classification' ): - eval_task = ModelEvaluationClassificationOp( + evaluation_automl_tabular_classification_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, slicing_specs=slicing_specs, - ) - ModelImportEvaluationOp( - classification_metrics=eval_task.outputs['evaluation_metrics'], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, - ) - - with kfp.dsl.Condition(prediction_type == 'forecasting', name='forecasting'): - eval_task = ModelEvaluationForecastingOp( - project=project, - location=location, - target_field_name=target_field_name, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - ModelImportEvaluationOp( - forecasting_metrics=eval_task.outputs['evaluation_metrics'], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) with kfp.dsl.Condition(prediction_type == 'regression', name='regression'): - eval_task = ModelEvaluationRegressionOp( + evaluation_automl_tabular_regression_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - ModelImportEvaluationOp( - regression_metrics=eval_task.outputs['evaluation_metrics'], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_unstructure_data_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_unstructure_data_pipeline.py index 89b3b4da43..4148df50de 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_unstructure_data_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_automl_unstructure_data_pipeline.py @@ -12,22 +12,194 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, List, NamedTuple + +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components._implementation.model import GetVertexModelOp from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp from google_cloud_pipeline_components._implementation.model_evaluation import TargetFieldDataRemoverOp +from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics +from google_cloud_pipeline_components.types.artifact_types import RegressionMetrics from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp -from google_cloud_pipeline_components.v1.model_evaluation.forecasting_component import model_evaluation_forecasting as ModelEvaluationForecastingOp from google_cloud_pipeline_components.v1.model_evaluation.regression_component import model_evaluation_regression as ModelEvaluationRegressionOp import kfp from kfp import dsl -@kfp.dsl.pipeline(name='evaluation-automl-unstructure-data-pipeline') -def evaluation_automl_unstructure_data_pipeline( # pylint: disable=dangerous-default-value +@kfp.dsl.pipeline(name='evaluation-classification-pipeline') +def evaluation_automl_unstructure_data_classification_pipeline( # pylint: disable=dangerous-default-value + location: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic + evaluation_prediction_label_column: str = '', + evaluation_prediction_score_column: str = '', + evaluation_class_labels: List[str] = [], # pylint: disable=g-bare-generic + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-unstructured-data-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +) -> NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, +): + # fmt: off + """The evaluation pipeline with ground truth and no feature attribution for classification models. + + This pipeline is used for all classification unstructured AutoML models, including Text, Video, Image and Custom models. + + Args: + location: The GCP region that runs the pipeline components. + model_name: The Vertex model resource name to be imported and used for batch prediction. Formatted like projects/{project}/locations/{location}/models/{model} or projects/{project}/locations/{location}/models/{model}@{model_version_id_or_model_version_alias}. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See sample code in https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component For more details on configuring slices, see https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice. + evaluation_prediction_label_column: The column name of the field containing classes the model is scoring. Formatted to be able to find nested columns, delimited by `.`. + evaluation_prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + evaluation_class_labels: Required for classification prediction type. The list of class names for the target_field_name, in the same order they appear in a file in batch_predict_gcs_source_uris. For instance, if the target_field_name could be either `1` or `0`, then the class_labels input will be ["1", "0"]. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the project in which the PipelineJob is run. + + Returns: A Tuple of google.ClassificationMetrics artifact and the imported evaluation metrics resource name. + """ + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, + ) + + get_model_task = GetVertexModelOp(model_name=model_name) + + # Remove the ground truth from the given GCS data. + # This is required for many models as Vertex Batch Prediction can not have the + # ground truth in the data to run, but later the evaluation component requires + # the ground truth data. + target_field_data_remover_task = TargetFieldDataRemoverOp( + project=project, + location=location, + target_field_name=target_field_name, + gcs_source_uris=batch_predict_gcs_source_uris, + bigquery_source_uri=batch_predict_bigquery_source_uri, + instances_format=batch_predict_instances_format, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + ) + + # Run Batch Prediction. + batch_predict_task = ModelBatchPredictOp( + project=project, + location=location, + model=get_model_task.outputs['model'], + job_display_name=f'evaluation-batch-predict-{dsl.PIPELINE_JOB_ID_PLACEHOLDER}-{dsl.PIPELINE_TASK_ID_PLACEHOLDER}', + gcs_source_uris=target_field_data_remover_task.outputs[ + 'gcs_output_directory' + ], + bigquery_source_input_uri=target_field_data_remover_task.outputs[ + 'bigquery_output_table' + ], + instances_format=batch_predict_instances_format, + predictions_format=batch_predict_predictions_format, + gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, + bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + machine_type=batch_predict_machine_type, + starting_replica_count=batch_predict_starting_replica_count, + max_replica_count=batch_predict_max_replica_count, + encryption_spec_key_name=encryption_spec_key_name, + accelerator_type=batch_predict_accelerator_type, + accelerator_count=batch_predict_accelerator_count, + ) + + # Run evaluation for a classification model. + eval_task = ModelEvaluationClassificationOp( + project=project, + location=location, + class_labels=evaluation_class_labels, + prediction_label_column=evaluation_prediction_label_column, + prediction_score_column=evaluation_prediction_score_column, + target_field_name=target_field_name, + ground_truth_format=batch_predict_instances_format, + ground_truth_gcs_source=batch_predict_gcs_source_uris, + ground_truth_bigquery_source=batch_predict_bigquery_source_uri, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + slicing_specs=slicing_specs, + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + classification_metrics=eval_task.outputs['evaluation_metrics'], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline(name='evaluation-regression-pipeline') +def evaluation_automl_unstructure_data_regression_pipeline( # pylint: disable=dangerous-default-value project: str, location: str, - prediction_type: str, model_name: str, target_field_name: str, batch_predict_instances_format: str, @@ -41,9 +213,7 @@ def evaluation_automl_unstructure_data_pipeline( # pylint: disable=dangerous-de batch_predict_max_replica_count: int = 10, batch_predict_accelerator_type: str = '', batch_predict_accelerator_count: int = 0, - evaluation_prediction_label_column: str = '', evaluation_prediction_score_column: str = '', - evaluation_class_labels: list = [], # pylint: disable=g-bare-generic dataflow_machine_type: str = 'n1-standard-4', dataflow_max_num_workers: int = 5, dataflow_disk_size_gb: int = 50, @@ -51,134 +221,56 @@ def evaluation_automl_unstructure_data_pipeline( # pylint: disable=dangerous-de dataflow_subnetwork: str = '', dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-unstructured-data-pipeline-{{$.pipeline_job_uuid}}', force_runner_mode: str = '', +) -> NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, ): - """The evaluation pipeline with ground truth and no feature attribution. + # fmt: off + """The evaluation pipeline with ground truth and no feature attribution for. - This pipeline is used for all unstructured AutoML models, including Text, - Video, Image and Custom imported models. + regression models. + + This pipeline is used for all custom tabular regression models. Args: project: The GCP project that runs the pipeline components. location: The GCP region that runs the pipeline components. - prediction_type: The type of prediction the model is to produce. - "classification", "regression", or "forecasting". - model_name: The Vertex model resource name to be imported and used for batch - prediction. Formatted like - projects/{project}/locations/{location}/models/{model} or - projects/{project}/locations/{location}/models/{model}@{model_version_id_or_model_version_alias}. - target_field_name: The target field's name. Formatted to be able to find - nested columns, delimited by ``.``. Prefixed with 'instance.' on the - component for Vertex Batch Prediction. - batch_predict_instances_format: The format in which instances are given, - must be one of the Model's supportedInputStorageFormats. For more details - about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_gcs_destination_output_uri: The Google Cloud Storage location - of the directory where the output is to be written to. In the given - directory a new directory is created. Its name is - ``prediction--``, where timestamp is - in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files - ``predictions_0001.``, ``predictions_0002.``, ..., - ``predictions_N.`` are created where ```` depends on - chosen ``predictions_format``, and N may equal 0001 and depends on the - total number of successfully predicted instances. If the Model has both - ``instance`` and ``prediction`` schemata defined then each such file - contains predictions as per the ``predictions_format``. If prediction for - any instance failed (partially or completely), then an additional - ``errors_0001.``, ``errors_0002.``,..., - ``errors_N.`` files are created (N depends on total number of - failed predictions). These files contain the failed instances, as per - their schema, followed by an additional ``error`` field which as value has - ``google.rpc.Status`` containing only ``code`` and ``message`` fields. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your - instances data to run batch prediction on. The instances data should also - contain the ground truth (target) data, used for evaluation. May contain - wildcards. For more information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For - more details about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to - run batch prediction on. May contain wildcards. For more details about - this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_predictions_format: The format in which Vertex AI gives the - predictions. Must be one of the Model's supportedOutputStorageFormats. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_bigquery_destination_output_uri: The BigQuery project location - where the output is to be written to. In the given project a new dataset - is created with name ``prediction__`` - where is made BigQuery-dataset-name compatible (for example, most special - characters become underscores), and timestamp is in - YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two - tables will be created, ``predictions``, and ``errors``. If the Model has - both ``instance`` and ``prediction`` schemata defined then the tables have - columns as follows: The ``predictions`` table contains instances for which - the prediction succeeded, it has columns as per a concatenation of the - Model's instance and prediction schemata. The ``errors`` table contains - rows for which the prediction has failed, it has instance columns, as per - the instance schema, followed by a single "errors" column, which as values - has ````google.rpc.Status`` ``__ represented as a STRUCT, and - containing only ``code`` and ``message``. For more details about this - output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_machine_type: The type of machine for running batch prediction - on dedicated resources. If the Model supports DEDICATED_RESOURCES this - config may be provided (and the job will use these resources). If the - Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. - For more details about the BatchDedicatedResources, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. - For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_starting_replica_count: The number of machine replicas used at - the start of the batch operation. If not set, Vertex AI decides starting - number, not greater than ``max_replica_count``. Only used if - ``machine_type`` is set. - batch_predict_max_replica_count: The maximum number of machine replicas the - batch operation may be scaled to. Only used if ``machine_type`` is set. - batch_predict_accelerator_type: The type of accelerator(s) that may be - attached to the machine as per ``batch_predict_accelerator_count``. Only - used if ``batch_predict_machine_type`` is set. For more details about the - machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_accelerator_count: The number of accelerators to attach to the - ``batch_predict_machine_type``. Only used if - ``batch_predict_machine_type`` is set. - evaluation_prediction_label_column: The column name of the field containing - classes the model is scoring. Formatted to be able to find nested columns, - delimited by ``.``. - evaluation_prediction_score_column: The column name of the field containing - batch prediction scores. Formatted to be able to find nested columns, - delimited by ``.``. - evaluation_class_labels: Required for classification prediction type. The - list of class names for the target_field_name, in the same order they - appear in a file in batch_predict_gcs_source_uris. For instance, if the - target_field_name could be either ``1`` or ``0``, then the class_labels - input will be ["1", "0"]. + model_name: The Vertex model resource name to be imported and used for batch prediction. Formatted like projects/{project}/locations/{location}/models/{model} or projects/{project}/locations/{location}/models/{model}@{model_version_id_or_model_version_alias}. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + evaluation_prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. dataflow_machine_type: The Dataflow machine type for evaluation components. - dataflow_max_num_workers: The max number of Dataflow workers for evaluation - components. - dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation - components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. dataflow_service_account: Custom service account to run Dataflow jobs. - dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty - the default subnetwork will be used. Example: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow workers use public IP - addresses. - encryption_spec_key_name: Customer-managed encryption key options. If set, - resources created by this pipeline will be encrypted with the provided - encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Indicate the runner mode to use forcely. Valid options - are ``Dataflow`` and ``DirectRunner``. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + + Returns: A Tuple of google.RegressionMetrics artifact and the imported evaluation metrics resource name. """ - evaluation_display_name = 'evaluation_automl_unstructure_data_pipeline' + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, + ) + get_model_task = GetVertexModelOp(model_name=model_name) # Remove the ground truth from the given GCS data. @@ -223,113 +315,178 @@ def evaluation_automl_unstructure_data_pipeline( # pylint: disable=dangerous-de accelerator_count=batch_predict_accelerator_count, ) - # Run evaluation based on prediction type. - # After, import the model evaluations to the Vertex model. + # Run evaluation for a regression model. + eval_task = ModelEvaluationRegressionOp( + project=project, + location=location, + target_field_name=target_field_name, + ground_truth_format=batch_predict_instances_format, + ground_truth_gcs_source=batch_predict_gcs_source_uris, + ground_truth_bigquery_source=batch_predict_bigquery_source_uri, + prediction_score_column=evaluation_prediction_score_column, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + regression_metrics=eval_task.outputs['evaluation_metrics'], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline(name='evaluation-pipeline') +def evaluation_automl_unstructure_data_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + prediction_type: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic + evaluation_prediction_label_column: str = '', + evaluation_prediction_score_column: str = '', + evaluation_class_labels: List[str] = [], # pylint: disable=g-bare-generic + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-automl-unstructured-data-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', +): + # fmt: off + """The evaluation pipeline with ground truth and no feature attribution. + + This pipeline is used for all unstructured AutoML models, including Text, + Video, Image and Custom models. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + model_name: The Vertex model resource name to be imported and used for batch prediction. Formatted like projects/{project}/locations/{location}/models/{model} or projects/{project}/locations/{location}/models/{model}@{model_version_id_or_model_version_alias}. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See sample code in https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component For more details on configuring slices, see https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice. + evaluation_prediction_label_column: The column name of the field containing classes the model is scoring. Formatted to be able to find nested columns, delimited by `.`. + evaluation_prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + evaluation_class_labels: Required for classification prediction type. The list of class names for the target_field_name, in the same order they appear in a file in batch_predict_gcs_source_uris. For instance, if the target_field_name could be either `1` or `0`, then the class_labels input will be ["1", "0"]. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + """ + # fmt: on with kfp.dsl.Condition( prediction_type == 'classification', name='classification' ): - eval_task = ModelEvaluationClassificationOp( - project=project, - location=location, - class_labels=evaluation_class_labels, - prediction_label_column=evaluation_prediction_label_column, - prediction_score_column=evaluation_prediction_score_column, - target_field_name=target_field_name, - ground_truth_format=batch_predict_instances_format, - ground_truth_gcs_source=batch_predict_gcs_source_uris, - ground_truth_bigquery_source=batch_predict_bigquery_source_uri, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - ModelImportEvaluationOp( - classification_metrics=eval_task.outputs['evaluation_metrics'], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, - ) - - with kfp.dsl.Condition(prediction_type == 'forecasting', name='forecasting'): - eval_task = ModelEvaluationForecastingOp( + evaluation_automl_unstructure_data_classification_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - ground_truth_format=batch_predict_instances_format, - ground_truth_gcs_source=batch_predict_gcs_source_uris, - ground_truth_bigquery_source=batch_predict_bigquery_source_uri, - prediction_score_column=evaluation_prediction_score_column, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + slicing_specs=slicing_specs, + evaluation_prediction_label_column=evaluation_prediction_label_column, + evaluation_prediction_score_column=evaluation_prediction_score_column, + evaluation_class_labels=evaluation_class_labels, dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - ModelImportEvaluationOp( - forecasting_metrics=eval_task.outputs['evaluation_metrics'], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) with kfp.dsl.Condition(prediction_type == 'regression', name='regression'): - eval_task = ModelEvaluationRegressionOp( + evaluation_automl_unstructure_data_regression_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - ground_truth_format=batch_predict_instances_format, - ground_truth_gcs_source=batch_predict_gcs_source_uris, - ground_truth_bigquery_source=batch_predict_bigquery_source_uri, - prediction_score_column=evaluation_prediction_score_column, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + evaluation_prediction_score_column=evaluation_prediction_score_column, dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - ModelImportEvaluationOp( - regression_metrics=eval_task.outputs['evaluation_metrics'], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_feature_attribution_pipeline.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_feature_attribution_pipeline.py index 87f0d9666e..27c34f43b2 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_feature_attribution_pipeline.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/evaluation_feature_attribution_pipeline.py @@ -12,42 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, Dict, List, NamedTuple + +from google_cloud_pipeline_components import _placeholders from google_cloud_pipeline_components._implementation.model import GetVertexModelOp -from google_cloud_pipeline_components._implementation.model_evaluation import EvaluationDataSamplerOp +from google_cloud_pipeline_components._implementation.model_evaluation import FeatureAttributionGraphComponentOp from google_cloud_pipeline_components._implementation.model_evaluation import ModelImportEvaluationOp from google_cloud_pipeline_components._implementation.model_evaluation import TargetFieldDataRemoverOp -from google_cloud_pipeline_components.preview.model_evaluation import ModelEvaluationFeatureAttributionOp +from google_cloud_pipeline_components.types.artifact_types import ClassificationMetrics +from google_cloud_pipeline_components.types.artifact_types import RegressionMetrics from google_cloud_pipeline_components.v1.batch_predict_job import ModelBatchPredictOp from google_cloud_pipeline_components.v1.model_evaluation.classification_component import model_evaluation_classification as ModelEvaluationClassificationOp -from google_cloud_pipeline_components.v1.model_evaluation.forecasting_component import model_evaluation_forecasting as ModelEvaluationForecastingOp from google_cloud_pipeline_components.v1.model_evaluation.regression_component import model_evaluation_regression as ModelEvaluationRegressionOp import kfp -@kfp.dsl.pipeline(name='evaluation-feature-attribution-pipeline') -def evaluation_feature_attribution_pipeline( # pylint: disable=dangerous-default-value - project: str, +@kfp.dsl.pipeline(name='evaluation-feature-attribution-classification-pipeline') +def evaluation_feature_attribution_classification_pipeline( # pylint: disable=dangerous-default-value location: str, - prediction_type: str, model_name: str, target_field_name: str, batch_predict_instances_format: str, batch_predict_gcs_destination_output_uri: str, - batch_predict_gcs_source_uris: list = [], # pylint: disable=g-bare-generic + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic batch_predict_bigquery_source_uri: str = '', batch_predict_predictions_format: str = 'jsonl', batch_predict_bigquery_destination_output_uri: str = '', batch_predict_machine_type: str = 'n1-standard-16', batch_predict_starting_replica_count: int = 5, batch_predict_max_replica_count: int = 10, - batch_predict_explanation_metadata: dict = {}, # pylint: disable=g-bare-generic - batch_predict_explanation_parameters: dict = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_metadata: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: Dict[str, Any] = {}, # pylint: disable=g-bare-generic batch_predict_explanation_data_sample_size: int = 10000, batch_predict_accelerator_type: str = '', batch_predict_accelerator_count: int = 0, + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic evaluation_prediction_label_column: str = '', evaluation_prediction_score_column: str = '', - evaluation_class_labels: list = [], # pylint: disable=g-bare-generic + evaluation_class_labels: List[str] = [], # pylint: disable=g-bare-generic dataflow_machine_type: str = 'n1-standard-4', dataflow_max_num_workers: int = 5, dataflow_disk_size_gb: int = 50, @@ -55,156 +57,64 @@ def evaluation_feature_attribution_pipeline( # pylint: disable=dangerous-defaul dataflow_subnetwork: str = '', dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-feature-attribution-pipeline-{{$.pipeline_job_uuid}}', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, +) -> NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, ): - """The evaluation AutoML tabular pipeline with feature attribution. + # fmt: off + """The evaluation custom tabular pipeline with feature attribution for classification models. - This pipeline guarantees support for AutoML Tabular models that contain a - valid explanation_spec. This pipeline does not include the - target_field_data_remover component, which is needed for many tabular custom - models. + This pipeline gives support for custom models that contain a valid explanation_spec. This pipeline includes the target_field_data_remover component, which is needed for many tabular custom models. Args: - project: The GCP project that runs the pipeline components. location: The GCP region that runs the pipeline components. - prediction_type: The type of prediction the model is to produce. - "classification", "regression", or "forecasting". - model_name: The Vertex model resource name to be imported and used for batch - prediction. - target_field_name: The target field's name. Formatted to be able to find - nested columns, delimited by ``.``. Prefixed with 'instance.' on the - component for Vertex Batch Prediction. - batch_predict_instances_format: The format in which instances are given, - must be one of the Model's supportedInputStorageFormats. For more details - about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_gcs_destination_output_uri: The Google Cloud Storage location - of the directory where the output is to be written to. In the given - directory a new directory is created. Its name is - ``prediction--``, where timestamp is - in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files - ``predictions_0001.``, ``predictions_0002.``, ..., - ``predictions_N.`` are created where ```` depends on - chosen ``predictions_format``, and N may equal 0001 and depends on the - total number of successfully predicted instances. If the Model has both - ``instance`` and ``prediction`` schemata defined then each such file - contains predictions as per the ``predictions_format``. If prediction for - any instance failed (partially or completely), then an additional - ``errors_0001.``, ``errors_0002.``,..., - ``errors_N.`` files are created (N depends on total number of - failed predictions). These files contain the failed instances, as per - their schema, followed by an additional ``error`` field which as value has - ``google.rpc.Status`` containing only ``code`` and ``message`` fields. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your - instances data to run batch prediction on. The instances data should also - contain the ground truth (target) data, used for evaluation. May contain - wildcards. For more information on wildcards, see - https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For - more details about this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to - run batch prediction on. May contain wildcards. For more details about - this input config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. - batch_predict_predictions_format: The format in which Vertex AI gives the - predictions. Must be one of the Model's supportedOutputStorageFormats. For - more details about this output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_bigquery_destination_output_uri: The BigQuery project location - where the output is to be written to. In the given project a new dataset - is created with name ``prediction__`` - where is made BigQuery-dataset-name compatible (for example, most special - characters become underscores), and timestamp is in - YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two - tables will be created, ``predictions``, and ``errors``. If the Model has - both ``instance`` and ``prediction`` schemata defined then the tables have - columns as follows: The ``predictions`` table contains instances for which - the prediction succeeded, it has columns as per a concatenation of the - Model's instance and prediction schemata. The ``errors`` table contains - rows for which the prediction has failed, it has instance columns, as per - the instance schema, followed by a single "errors" column, which as values - has ````google.rpc.Status`` ``__ represented as a STRUCT, and - containing only ``code`` and ``message``. For more details about this - output config, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. - batch_predict_machine_type: The type of machine for running batch prediction - on dedicated resources. If the Model supports DEDICATED_RESOURCES this - config may be provided (and the job will use these resources). If the - Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. - For more details about the BatchDedicatedResources, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. - For more details about the machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_starting_replica_count: The number of machine replicas used at - the start of the batch operation. If not set, Vertex AI decides starting - number, not greater than ``max_replica_count``. Only used if - ``machine_type`` is set. - batch_predict_max_replica_count: The maximum number of machine replicas the - batch operation may be scaled to. Only used if ``machine_type`` is set. - batch_predict_explanation_metadata: Explanation metadata configuration for - this BatchPredictionJob. Can be specified only if ``generate_explanation`` - is set to ``True``. This value overrides the value of - ``Model.explanation_metadata``. All fields of ``explanation_metadata`` are - optional in the request. If a field of the ``explanation_metadata`` object - is not populated, the corresponding field of the - ``Model.explanation_metadata`` object is inherited. For more details, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. - batch_predict_explanation_parameters: Parameters to configure explaining for - Model's predictions. Can be specified only if ``generate_explanation`` is - set to ``True``. This value overrides the value of - ``Model.explanation_parameters``. All fields of ``explanation_parameters`` - are optional in the request. If a field of the ``explanation_parameters`` - object is not populated, the corresponding field of the - ``Model.explanation_parameters`` object is inherited. For more details, - see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. - batch_predict_explanation_data_sample_size: Desired size to downsample the - input dataset that will then be used for batch explanation. - batch_predict_accelerator_type: The type of accelerator(s) that may be - attached to the machine as per ``batch_predict_accelerator_count``. Only - used if ``batch_predict_machine_type`` is set. For more details about the - machine spec, see - https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec - batch_predict_accelerator_count: The number of accelerators to attach to the - ``batch_predict_machine_type``. Only used if - ``batch_predict_machine_type`` is set. - evaluation_prediction_label_column: The column name of the field containing - classes the model is scoring. Formatted to be able to find nested columns, - delimited by ``.``. - evaluation_prediction_score_column: The column name of the field containing - batch prediction scores. Formatted to be able to find nested columns, - delimited by ``.``. - evaluation_class_labels: Required for classification prediction type. The - list of class names for the target_field_name, in the same order they - appear in a file in batch_predict_gcs_source_uris. For instance, if the - target_field_name could be either ``1`` or ``0``, then the class_labels - input will be ["1", "0"]. + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See [sample code](https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component) and more details on [configuring slices](https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice). + evaluation_prediction_label_column: The column name of the field containing classes the model is scoring. Formatted to be able to find nested columns, delimited by `.`. + evaluation_prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + evaluation_class_labels: Required for classification prediction type. The list of class names for the target_field_name, in the same order they appear in a file in batch_predict_gcs_source_uris. For instance, if the target_field_name could be either `1` or `0`, then the class_labels input will be ["1", "0"]. dataflow_machine_type: The Dataflow machine type for evaluation components. - dataflow_max_num_workers: The max number of Dataflow workers for evaluation - components. - dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation - components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. dataflow_service_account: Custom service account to run Dataflow jobs. - dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty - the default subnetwork will be used. Example: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow workers use public IP - addresses. - encryption_spec_key_name: Customer-managed encryption key options. If set, - resources created by this pipeline will be encrypted with the provided - encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Indicate the runner mode to use forcely. Valid options - are ``Dataflow`` and ``DirectRunner``. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + project: The GCP project that runs the pipeline components. Defaults to the project in which the PipelineJob is run. + + Returns: A google.ClassificationMetrics artifact. """ - evaluation_display_name = 'evaluation-feature-attribution-pipeline' + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=ClassificationMetrics, + evaluation_resource_name=str, + ) + get_model_task = GetVertexModelOp(model_name=model_name) - # Remove the ground truth from the given GCS data. + # Remove the ground truth from the given GCS or BQ data. # This is required for many models as Vertex Batch Prediction can not have the # ground truth in the data to run, but later the evaluation component requires # the ground truth data. @@ -246,36 +156,209 @@ def evaluation_feature_attribution_pipeline( # pylint: disable=dangerous-defaul accelerator_count=batch_predict_accelerator_count, ) - # Run the Batch Explain process (sampler -> batch explanation). - data_sampler_task = EvaluationDataSamplerOp( + # Run feature attribution steps. + feature_attribution_graph = FeatureAttributionGraphComponentOp( project=project, location=location, - gcs_source_uris=target_field_data_remover_task.outputs[ + prediction_type='classification', + vertex_model=get_model_task.outputs['model'], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=target_field_data_remover_task.outputs[ 'gcs_output_directory' ], - bigquery_source_uri=target_field_data_remover_task.outputs[ + batch_predict_bigquery_source_uri=target_field_data_remover_task.outputs[ 'bigquery_output_table' ], + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + dataflow_machine_type=dataflow_machine_type, + dataflow_max_num_workers=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + ) + + # Run evaluation for a classification model. + eval_task = ModelEvaluationClassificationOp( + project=project, + location=location, + class_labels=evaluation_class_labels, + prediction_label_column=evaluation_prediction_label_column, + prediction_score_column=evaluation_prediction_score_column, + target_field_name=target_field_name, + ground_truth_format=batch_predict_instances_format, + ground_truth_gcs_source=batch_predict_gcs_source_uris, + ground_truth_bigquery_source=batch_predict_bigquery_source_uri, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + slicing_specs=slicing_specs, + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + classification_metrics=eval_task.outputs['evaluation_metrics'], + feature_attributions=feature_attribution_graph.outputs[ + 'feature_attributions' + ], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline(name='evaluation-feature-attribution-regression-pipeline') +def evaluation_feature_attribution_regression_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_explanation_metadata: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_data_sample_size: int = 10000, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + evaluation_prediction_score_column: str = '', + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-feature-attribution-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', +) -> NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, +): + # fmt: off + """The evaluation custom tabular pipeline with feature attribution for. + + regression models. + + This pipeline gives support for custom models that contain a + valid explanation_spec. This pipeline includes the target_field_data_remover + component, which is needed for many tabular custom models. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + evaluation_prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + + Returns: + A google.RegressionMetrics artifact. + """ + # fmt: on + outputs = NamedTuple( + 'outputs', + evaluation_metrics=RegressionMetrics, + evaluation_resource_name=str, + ) + + get_model_task = GetVertexModelOp(model_name=model_name) + + # Remove the ground truth from the given GCS or BQ data. + # This is required for many models as Vertex Batch Prediction can not have the + # ground truth in the data to run, but later the evaluation component requires + # the ground truth data. + target_field_data_remover_task = TargetFieldDataRemoverOp( + project=project, + location=location, + target_field_name=target_field_name, + gcs_source_uris=batch_predict_gcs_source_uris, + bigquery_source_uri=batch_predict_bigquery_source_uri, instances_format=batch_predict_instances_format, - sample_size=batch_predict_explanation_data_sample_size, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, force_runner_mode=force_runner_mode, ) - batch_explain_task = ModelBatchPredictOp( + + # Run Batch Prediction. + batch_predict_task = ModelBatchPredictOp( project=project, location=location, model=get_model_task.outputs['model'], - job_display_name='model-registry-batch-explain-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', - gcs_source_uris=data_sampler_task.outputs['gcs_output_directory'], - bigquery_source_input_uri=data_sampler_task.outputs[ + job_display_name='model-registry-batch-predict-evaluation-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}', + gcs_source_uris=target_field_data_remover_task.outputs[ + 'gcs_output_directory' + ], + bigquery_source_input_uri=target_field_data_remover_task.outputs[ 'bigquery_output_table' ], instances_format=batch_predict_instances_format, predictions_format=batch_predict_predictions_format, gcs_destination_output_uri_prefix=batch_predict_gcs_destination_output_uri, bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, - generate_explanation=True, - explanation_parameters=batch_predict_explanation_parameters, - explanation_metadata=batch_predict_explanation_metadata, machine_type=batch_predict_machine_type, starting_replica_count=batch_predict_starting_replica_count, max_replica_count=batch_predict_max_replica_count, @@ -284,182 +367,228 @@ def evaluation_feature_attribution_pipeline( # pylint: disable=dangerous-defaul accelerator_count=batch_predict_accelerator_count, ) - # Run evaluation based on prediction type and feature attribution component. - # After, import the model evaluations to the Vertex model. + # Run feature attribution steps. + feature_attribution_graph = FeatureAttributionGraphComponentOp( + project=project, + location=location, + prediction_type='regression', + vertex_model=get_model_task.outputs['model'], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=target_field_data_remover_task.outputs[ + 'gcs_output_directory' + ], + batch_predict_bigquery_source_uri=target_field_data_remover_task.outputs[ + 'bigquery_output_table' + ], + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + dataflow_machine_type=dataflow_machine_type, + dataflow_max_num_workers=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + ) + + # Run evaluation for a regression model. + eval_task = ModelEvaluationRegressionOp( + project=project, + location=location, + target_field_name=target_field_name, + ground_truth_format=batch_predict_instances_format, + ground_truth_gcs_source=batch_predict_gcs_source_uris, + ground_truth_bigquery_source=batch_predict_bigquery_source_uri, + prediction_score_column=evaluation_prediction_score_column, + predictions_format=batch_predict_predictions_format, + predictions_gcs_source=batch_predict_task.outputs['gcs_output_directory'], + predictions_bigquery_source=batch_predict_task.outputs[ + 'bigquery_output_table' + ], + dataflow_machine_type=dataflow_machine_type, + dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_disk_size_gb=dataflow_disk_size_gb, + dataflow_service_account=dataflow_service_account, + dataflow_subnetwork=dataflow_subnetwork, + dataflow_use_public_ips=dataflow_use_public_ips, + encryption_spec_key_name=encryption_spec_key_name, + force_runner_mode=force_runner_mode, + model=get_model_task.outputs['model'], + ) + + # Import the evaluation result to Vertex AI. + import_evaluation_task = ModelImportEvaluationOp( + regression_metrics=eval_task.outputs['evaluation_metrics'], + feature_attributions=feature_attribution_graph.outputs[ + 'feature_attributions' + ], + model=get_model_task.outputs['model'], + dataset_type=batch_predict_instances_format, + dataset_path=batch_predict_bigquery_source_uri, + dataset_paths=batch_predict_gcs_source_uris, + display_name=evaluation_display_name, + ) + + return outputs( + evaluation_metrics=eval_task.outputs['evaluation_metrics'], + evaluation_resource_name=import_evaluation_task.outputs[ + 'evaluation_resource_name' + ], + ) + + +@kfp.dsl.pipeline(name='evaluation-feature-attribution-pipeline') +def evaluation_feature_attribution_pipeline( # pylint: disable=dangerous-default-value + project: str, + location: str, + prediction_type: str, + model_name: str, + target_field_name: str, + batch_predict_instances_format: str, + batch_predict_gcs_destination_output_uri: str, + batch_predict_gcs_source_uris: List[str] = [], # pylint: disable=g-bare-generic + batch_predict_bigquery_source_uri: str = '', + batch_predict_predictions_format: str = 'jsonl', + batch_predict_bigquery_destination_output_uri: str = '', + batch_predict_machine_type: str = 'n1-standard-16', + batch_predict_starting_replica_count: int = 5, + batch_predict_max_replica_count: int = 10, + batch_predict_explanation_metadata: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_parameters: Dict[str, Any] = {}, # pylint: disable=g-bare-generic + batch_predict_explanation_data_sample_size: int = 10000, + batch_predict_accelerator_type: str = '', + batch_predict_accelerator_count: int = 0, + slicing_specs: List[Any] = [], # pylint: disable=g-bare-generic + evaluation_prediction_label_column: str = '', + evaluation_prediction_score_column: str = '', + evaluation_class_labels: List[str] = [], # pylint: disable=g-bare-generic + dataflow_machine_type: str = 'n1-standard-4', + dataflow_max_num_workers: int = 5, + dataflow_disk_size_gb: int = 50, + dataflow_service_account: str = '', + dataflow_subnetwork: str = '', + dataflow_use_public_ips: bool = True, + encryption_spec_key_name: str = '', + evaluation_display_name: str = 'evaluation-feature-attribution-pipeline-{{$.pipeline_job_uuid}}', + force_runner_mode: str = '', +): + # fmt: off + """The evaluation custom tabular pipeline with feature attribution. + + This pipeline gives support for custom models that contain a + valid explanation_spec. This pipeline includes the target_field_data_remover + component, which is needed for many tabular custom models. + + Args: + project: The GCP project that runs the pipeline components. + location: The GCP region that runs the pipeline components. + prediction_type: The type of prediction the model is to produce. "classification" or "regression". + model_name: The Vertex model resource name to be imported and used for batch prediction. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + batch_predict_instances_format: The format in which instances are given, must be one of the Model's supportedInputStorageFormats. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_gcs_destination_output_uri: The Google Cloud Storage location of the directory where the output is to be written to. In the given directory a new directory is created. Its name is `prediction--`, where timestamp is in YYYY-MM-DDThh:mm:ss.sssZ ISO-8601 format. Inside of it files `predictions_0001.`, `predictions_0002.`, ..., `predictions_N.` are created where `` depends on chosen `predictions_format`, and N may equal 0001 and depends on the total number of successfully predicted instances. If the Model has both `instance` and `prediction` schemata defined then each such file contains predictions as per the `predictions_format`. If prediction for any instance failed (partially or completely), then an additional `errors_0001.`, `errors_0002.`,..., `errors_N.` files are created (N depends on total number of failed predictions). These files contain the failed instances, as per their schema, followed by an additional `error` field which as value has `google.rpc.Status` containing only `code` and `message` fields. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_gcs_source_uris: Google Cloud Storage URI(-s) to your instances data to run batch prediction on. The instances data should also contain the ground truth (target) data, used for evaluation. May contain wildcards. For more information on wildcards, see https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_bigquery_source_uri: Google BigQuery URI to your instances to run batch prediction on. May contain wildcards. For more details about this input config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#InputConfig. + batch_predict_predictions_format: The format in which Vertex AI gives the predictions. Must be one of the Model's supportedOutputStorageFormats. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_bigquery_destination_output_uri: The BigQuery project location where the output is to be written to. In the given project a new dataset is created with name `prediction__` where is made BigQuery-dataset-name compatible (for example, most special characters become underscores), and timestamp is in YYYY_MM_DDThh_mm_ss_sssZ "based on ISO-8601" format. In the dataset two tables will be created, `predictions`, and `errors`. If the Model has both `instance` and `prediction` schemata defined then the tables have columns as follows: The `predictions` table contains instances for which the prediction succeeded, it has columns as per a concatenation of the Model's instance and prediction schemata. The `errors` table contains rows for which the prediction has failed, it has instance columns, as per the instance schema, followed by a single "errors" column, which as values has `google.rpc.Status` represented as a STRUCT, and containing only `code` and `message`. For more details about this output config, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#OutputConfig. + batch_predict_machine_type: The type of machine for running batch prediction on dedicated resources. If the Model supports DEDICATED_RESOURCES this config may be provided (and the job will use these resources). If the Model doesn't support AUTOMATIC_RESOURCES, this config must be provided. For more details about the BatchDedicatedResources, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.batchPredictionJobs#BatchDedicatedResources. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_starting_replica_count: The number of machine replicas used at the start of the batch operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. + batch_predict_max_replica_count: The maximum number of machine replicas the batch operation may be scaled to. Only used if `machine_type` is set. + batch_predict_explanation_metadata: Explanation metadata configuration for this BatchPredictionJob. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_metadata`. All fields of `explanation_metadata` are optional in the request. If a field of the `explanation_metadata` object is not populated, the corresponding field of the `Model.explanation_metadata` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#explanationmetadata. + batch_predict_explanation_parameters: Parameters to configure explaining for Model's predictions. Can be specified only if `generate_explanation` is set to `True`. This value overrides the value of `Model.explanation_parameters`. All fields of `explanation_parameters` are optional in the request. If a field of the `explanation_parameters` object is not populated, the corresponding field of the `Model.explanation_parameters` object is inherited. For more details, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/ExplanationSpec#ExplanationParameters. + batch_predict_explanation_data_sample_size: Desired size to downsample the input dataset that will then be used for batch explanation. + batch_predict_accelerator_type: The type of accelerator(s) that may be attached to the machine as per `batch_predict_accelerator_count`. Only used if `batch_predict_machine_type` is set. For more details about the machine spec, see https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec + batch_predict_accelerator_count: The number of accelerators to attach to the `batch_predict_machine_type`. Only used if `batch_predict_machine_type` is set. + slicing_specs: List of `google.cloud.aiplatform_v1.types.ModelEvaluationSlice.SlicingSpec`. When provided, compute metrics for each defined slice. See [sample code](https://cloud.google.com/vertex-ai/docs/pipelines/model-evaluation-component) and more details on [configuring slices](https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform_v1.types.ModelEvaluationSlice). + evaluation_prediction_label_column: The column name of the field containing classes the model is scoring. Formatted to be able to find nested columns, delimited by `.`. + evaluation_prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + evaluation_class_labels: Required for classification prediction type. The list of class names for the target_field_name, in the same order they appear in a file in batch_predict_gcs_source_uris. For instance, if the target_field_name could be either `1` or `0`, then the class_labels input will be ["1", "0"]. + dataflow_machine_type: The Dataflow machine type for evaluation components. + dataflow_max_num_workers: The max number of Dataflow workers for evaluation components. + dataflow_disk_size_gb: Dataflow worker's disk size in GB for evaluation components. + dataflow_service_account: Custom service account to run Dataflow jobs. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. Example: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + evaluation_display_name: The display name of the uploaded evaluation resource to the Vertex AI model. + force_runner_mode: Indicate the runner mode to use forcely. Valid options are `Dataflow` and `DirectRunner`. + """ + # fmt: on with kfp.dsl.Condition( prediction_type == 'classification', name='classification' ): - eval_task = ModelEvaluationClassificationOp( - project=project, - location=location, - class_labels=evaluation_class_labels, - prediction_label_column=evaluation_prediction_label_column, - prediction_score_column=evaluation_prediction_score_column, - target_field_name=target_field_name, - ground_truth_format=batch_predict_instances_format, - ground_truth_gcs_source=batch_predict_gcs_source_uris, - ground_truth_bigquery_source=batch_predict_bigquery_source_uri, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - feature_attribution_task = ModelEvaluationFeatureAttributionOp( - project=project, - location=location, - problem_type=prediction_type, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_explain_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_explain_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - ) - ModelImportEvaluationOp( - classification_metrics=eval_task.outputs['evaluation_metrics'], - feature_attributions=feature_attribution_task.outputs[ - 'feature_attributions' - ], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, - ) - - with kfp.dsl.Condition(prediction_type == 'forecasting', name='forecasting'): - eval_task = ModelEvaluationForecastingOp( + evaluation_feature_attribution_classification_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - ground_truth_format=batch_predict_instances_format, - ground_truth_gcs_source=batch_predict_gcs_source_uris, - ground_truth_bigquery_source=batch_predict_bigquery_source_uri, - prediction_score_column=evaluation_prediction_score_column, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + slicing_specs=slicing_specs, + evaluation_prediction_label_column=evaluation_prediction_label_column, + evaluation_prediction_score_column=evaluation_prediction_score_column, + evaluation_class_labels=evaluation_class_labels, dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - feature_attribution_task = ModelEvaluationFeatureAttributionOp( - project=project, - location=location, - problem_type=prediction_type, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_explain_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_explain_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - ) - ModelImportEvaluationOp( - forecasting_metrics=eval_task.outputs['evaluation_metrics'], - feature_attributions=feature_attribution_task.outputs[ - 'feature_attributions' - ], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) with kfp.dsl.Condition(prediction_type == 'regression', name='regression'): - eval_task = ModelEvaluationRegressionOp( + evaluation_feature_attribution_regression_pipeline( project=project, location=location, + model_name=model_name, target_field_name=target_field_name, - ground_truth_format=batch_predict_instances_format, - ground_truth_gcs_source=batch_predict_gcs_source_uris, - ground_truth_bigquery_source=batch_predict_bigquery_source_uri, - prediction_score_column=evaluation_prediction_score_column, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_predict_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_predict_task.outputs[ - 'bigquery_output_table' - ], + batch_predict_instances_format=batch_predict_instances_format, + batch_predict_gcs_destination_output_uri=batch_predict_gcs_destination_output_uri, + batch_predict_gcs_source_uris=batch_predict_gcs_source_uris, + batch_predict_bigquery_source_uri=batch_predict_bigquery_source_uri, + batch_predict_predictions_format=batch_predict_predictions_format, + batch_predict_bigquery_destination_output_uri=batch_predict_bigquery_destination_output_uri, + batch_predict_machine_type=batch_predict_machine_type, + batch_predict_starting_replica_count=batch_predict_starting_replica_count, + batch_predict_max_replica_count=batch_predict_max_replica_count, + batch_predict_explanation_metadata=batch_predict_explanation_metadata, + batch_predict_explanation_parameters=batch_predict_explanation_parameters, + batch_predict_explanation_data_sample_size=batch_predict_explanation_data_sample_size, + batch_predict_accelerator_type=batch_predict_accelerator_type, + batch_predict_accelerator_count=batch_predict_accelerator_count, + evaluation_prediction_score_column=evaluation_prediction_score_column, dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, + dataflow_max_num_workers=dataflow_max_num_workers, dataflow_disk_size_gb=dataflow_disk_size_gb, dataflow_service_account=dataflow_service_account, dataflow_subnetwork=dataflow_subnetwork, dataflow_use_public_ips=dataflow_use_public_ips, encryption_spec_key_name=encryption_spec_key_name, + evaluation_display_name=evaluation_display_name, force_runner_mode=force_runner_mode, - model=get_model_task.outputs['model'], - ) - feature_attribution_task = ModelEvaluationFeatureAttributionOp( - project=project, - location=location, - problem_type=prediction_type, - predictions_format=batch_predict_predictions_format, - predictions_gcs_source=batch_explain_task.outputs[ - 'gcs_output_directory' - ], - predictions_bigquery_source=batch_explain_task.outputs[ - 'bigquery_output_table' - ], - dataflow_machine_type=dataflow_machine_type, - dataflow_max_workers_num=dataflow_max_num_workers, - dataflow_disk_size_gb=dataflow_disk_size_gb, - dataflow_service_account=dataflow_service_account, - dataflow_subnetwork=dataflow_subnetwork, - dataflow_use_public_ips=dataflow_use_public_ips, - encryption_spec_key_name=encryption_spec_key_name, - force_runner_mode=force_runner_mode, - ) - ModelImportEvaluationOp( - regression_metrics=eval_task.outputs['evaluation_metrics'], - feature_attributions=feature_attribution_task.outputs[ - 'feature_attributions' - ], - model=get_model_task.outputs['model'], - dataset_type=batch_predict_instances_format, - dataset_path=batch_predict_bigquery_source_uri, - dataset_paths=batch_predict_gcs_source_uris, - display_name=evaluation_display_name, ) diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/forecasting_component.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/forecasting_component.py index 6a32a60586..ed133ce391 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/forecasting_component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/forecasting_component.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import version from google_cloud_pipeline_components.types.artifact_types import BQTable from google_cloud_pipeline_components.types.artifact_types import ForecastingMetrics from google_cloud_pipeline_components.types.artifact_types import VertexModel -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG from kfp import dsl from kfp.dsl import container_component @@ -24,7 +27,6 @@ def model_evaluation_forecasting( gcp_resources: dsl.OutputPath(str), evaluation_metrics: dsl.Output[ForecastingMetrics], - project: str, target_field_name: str, model: dsl.Input[VertexModel] = None, location: str = 'us-central1', @@ -32,10 +34,10 @@ def model_evaluation_forecasting( predictions_gcs_source: dsl.Input[dsl.Artifact] = None, predictions_bigquery_source: dsl.Input[BQTable] = None, ground_truth_format: str = 'jsonl', - ground_truth_gcs_source: list = [], + ground_truth_gcs_source: List[str] = [], ground_truth_bigquery_source: str = '', forecasting_type: str = 'point', - forecasting_quantiles: list = [], + forecasting_quantiles: List[float] = [], point_evaluation_quantile: float = 0.5, prediction_score_column: str = 'prediction.value', dataflow_service_account: str = '', @@ -47,9 +49,10 @@ def model_evaluation_forecasting( dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Computes a ``google.ForecastingMetrics`` Artifact, containing evaluation + """Computes a `google.ForecastingMetrics` Artifact, containing evaluation metrics given a model's prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation @@ -57,87 +60,38 @@ def model_evaluation_forecasting( Supports point forecasting and quantile forecasting for tabular data. Args: - project: Project to run evaluation container. location: Location for running the evaluation. - predictions_format: The file format for the batch - prediction results. ``jsonl``, ``csv``, and ``bigquery`` are the allowed - formats, from Vertex Batch Prediction. - predictions_gcs_source: An artifact with its - URI pointing toward a GCS directory with prediction or explanation files - to be used for this evaluation. For prediction results, the files should - be named "prediction.results-*". For explanation results, the files - should be named "explanation.results-*". - predictions_bigquery_source: BigQuery table - with prediction or explanation data to be used for this evaluation. For - prediction results, the table column should be named "predicted_*". - ground_truth_format: Required for custom tabular and non - tabular data. The file format for the ground truth files. ``jsonl``, - ``csv``, and ``bigquery`` are the allowed formats. - ground_truth_gcs_source: Required for custom - tabular and non tabular data. The GCS URIs representing where the ground - truth is located. Used to provide ground truth for each prediction - instance when they are not part of the batch prediction jobs prediction - instance. - ground_truth_bigquery_source: Required for custom tabular. - The BigQuery table URI representing where the ground truth is located. - Used to provide ground truth for each prediction instance when they are - not part of the batch prediction jobs prediction instance. - forecasting_type: The forecasting type being addressed by - this evaluation run. ``point`` and ``quantile`` are the supported types. - forecasting_quantiles: Required for a - ``quantile`` forecasting_type. The list of quantiles in the same order - appeared in the quantile prediction score column. - point_evaluation_quantile: Required for a ``quantile`` - forecasting_type. A quantile in the list of forecasting_quantiles that - will be used for point evaluation metrics. - target_field_name: The full name path of the features target field - in the predictions file. Formatted to be able to find nested columns, - delimited by ``.``. Alternatively referred to as the ground truth (or - ground_truth_column) field. - model: The Vertex model used for evaluation. Must be located in the same - region as the location argument. It is used to set the default - configurations for AutoML and custom-trained models. - prediction_score_column: The column name of the field - containing batch prediction scores. Formatted to be able to find nested - columns, delimited by ``.``. - dataflow_service_account: Service account to run the - Dataflow job. If not set, Dataflow will use the default worker service - account. For more details, see - https://cloud.google.com/dataflow/docs/concepts/secURIty-and-permissions#default_worker_service_account - dataflow_disk_size_gb: The disk size (in GB) of the machine - executing the evaluation run. - dataflow_machine_type: The machine type executing the - evaluation run. - dataflow_workers_num: The number of workers executing the - evaluation run. - dataflow_max_workers_num: The max number of workers - executing the evaluation run. - dataflow_subnetwork: Dataflow's fully qualified subnetwork - name, when empty the default subnetwork will be used. More - details: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow - workers use public IP addresses. - encryption_spec_key_name: Customer-managed encryption key options. - If set, resources created by this pipeline will be encrypted with the - provided encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - force_runner_mode: Flag to choose Beam runner. Valid options are ``DirectRunner`` - and ``Dataflow``. + predictions_format: The file format for the batch prediction results. `jsonl`, `csv`, and `bigquery` are the allowed formats, from Vertex Batch Prediction. + predictions_gcs_source: An artifact with its URI pointing toward a GCS directory with prediction or explanation files to be used for this evaluation. For prediction results, the files should be named "prediction.results-*". For explanation results, the files should be named "explanation.results-*". + predictions_bigquery_source: BigQuery table with prediction or explanation data to be used for this evaluation. For prediction results, the table column should be named "predicted_*". + ground_truth_format: Required for custom tabular and non tabular data. The file format for the ground truth files. `jsonl`, `csv`, and `bigquery` are the allowed formats. + ground_truth_gcs_source: Required for custom tabular and non tabular data. The GCS URIs representing where the ground truth is located. Used to provide ground truth for each prediction instance when they are not part of the batch prediction jobs prediction instance. + ground_truth_bigquery_source: Required for custom tabular. The BigQuery table URI representing where the ground truth is located. Used to provide ground truth for each prediction instance when they are not part of the batch prediction jobs prediction instance. + forecasting_type: The forecasting type being addressed by this evaluation run. `point` and `quantile` are the supported types. + forecasting_quantiles: Required for a `quantile` forecasting_type. The list of quantiles in the same order appeared in the quantile prediction score column. + point_evaluation_quantile: Required for a `quantile` forecasting_type. A quantile in the list of forecasting_quantiles that will be used for point evaluation metrics. + target_field_name: The full name path of the features target field in the predictions file. Formatted to be able to find nested columns, delimited by `.`. Alternatively referred to as the ground truth (or ground_truth_column) field. + model: The Vertex model used for evaluation. Must be located in the same region as the location argument. It is used to set the default configurations for AutoML and custom-trained models. + prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + dataflow_service_account: Service account to run the Dataflow job. If not set, Dataflow will use the default worker service account. For more details, see https://cloud.google.com/dataflow/docs/concepts/secURIty-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. + dataflow_machine_type: The machine type executing the evaluation run. + dataflow_workers_num: The number of workers executing the evaluation run. + dataflow_max_workers_num: The max number of workers executing the evaluation run. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + force_runner_mode: Flag to choose Beam runner. Valid options are `DirectRunner` and `Dataflow`. + project: Project to run evaluation container. Defaults to the project in which the PipelineJob is run. + Returns: - evaluation_metrics: - ``google.ForecastingMetrics`` representing the forecasting - evaluation metrics in GCS. - gcp_resources: Serialized gcp_resources proto tracking the Dataflow - job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + evaluation_metrics: `google.ForecastingMetrics` representing the forecasting evaluation metrics in GCS. + gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: off return dsl.ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/regression_component.py b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/regression_component.py index 8cc52948bf..e989181814 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/regression_component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/model_evaluation/regression_component.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List + +from google_cloud_pipeline_components import _placeholders +from google_cloud_pipeline_components._implementation.model_evaluation import version from google_cloud_pipeline_components.types.artifact_types import BQTable from google_cloud_pipeline_components.types.artifact_types import RegressionMetrics from google_cloud_pipeline_components.types.artifact_types import VertexModel -from google_cloud_pipeline_components.v1.model_evaluation.version import EVAL_IMAGE_TAG from kfp import dsl from kfp.dsl import container_component @@ -24,7 +27,6 @@ def model_evaluation_regression( gcp_resources: dsl.OutputPath(str), evaluation_metrics: dsl.Output[RegressionMetrics], - project: str, target_field_name: str, model: dsl.Input[VertexModel] = None, location: str = 'us-central1', @@ -32,7 +34,7 @@ def model_evaluation_regression( predictions_gcs_source: dsl.Input[dsl.Artifact] = None, predictions_bigquery_source: dsl.Input[BQTable] = None, ground_truth_format: str = 'jsonl', - ground_truth_gcs_source: list = [], + ground_truth_gcs_source: List[str] = [], ground_truth_bigquery_source: str = '', prediction_score_column: str = 'prediction.value', dataflow_service_account: str = '', @@ -44,9 +46,10 @@ def model_evaluation_regression( dataflow_use_public_ips: bool = True, encryption_spec_key_name: str = '', force_runner_mode: str = '', + project: str = _placeholders.PROJECT_ID_PLACEHOLDER, ): # fmt: off - """Computes a ``google.RegressionMetrics`` Artifact, containing evaluation + """Computes a `google.RegressionMetrics` Artifact, containing evaluation metrics given a model's prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation @@ -54,79 +57,34 @@ def model_evaluation_regression( Supports regression for tabular data. Args: - project: Project to run evaluation container. location: Location for running the evaluation. - predictions_format: The file format for the batch - prediction results. ``jsonl``, ``csv``, and ``bigquery`` are the allowed - formats, from Vertex Batch Prediction. - predictions_gcs_source: An artifact with its - URI pointing toward a GCS directory with prediction or explanation files - to be used for this evaluation. For prediction results, the files should - be named "prediction.results-*". For explanation results, the files - should be named "explanation.results-*". - predictions_bigquery_source: BigQuery table - with prediction or explanation data to be used for this evaluation. For - prediction results, the table column should be named "predicted_*". - ground_truth_format: Required for custom tabular and non - tabular data. The file format for the ground truth files. ``jsonl``, - ``csv``, and ``bigquery`` are the allowed formats. - ground_truth_gcs_source: Required for custom - tabular and non tabular data. The GCS URIs representing where the ground - truth is located. Used to provide ground truth for each prediction - instance when they are not part of the batch prediction jobs prediction - instance. - ground_truth_bigquery_source: Required for custom tabular. - The BigQuery table URI representing where the ground truth is located. - Used to provide ground truth for each prediction instance when they are - not part of the batch prediction jobs prediction instance. - target_field_name: The target field's name. Formatted to be able to find - nested columns, delimited by ``.``. Prefixed with 'instance.' on the - component for Vertex Batch Prediction. - model: The Vertex model used for evaluation. Must be located in the same - region as the location argument. It is used to set the default - configurations for AutoML and custom-trained models. - prediction_score_column: The column name of the field - containing batch prediction scores. Formatted to be able to find nested - columns, delimited by ``.``. - dataflow_service_account: Service account to run the - Dataflow job. If not set, Dataflow will use the default worker service - account. For more details, see - https://cloud.google.com/dataflow/docs/concepts/secURIty-and-permissions#default_worker_service_account - dataflow_disk_size_gb: The disk size (in GB) of the machine - executing the evaluation run. - dataflow_machine_type: The machine type executing the - evaluation run. - dataflow_workers_num: The number of workers executing the - evaluation run. - dataflow_max_workers_num: The max number of workers - executing the evaluation run. - dataflow_subnetwork: Dataflow's fully qualified subnetwork - name, when empty the default subnetwork will be used. More - details: - https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications - dataflow_use_public_ips: Specifies whether Dataflow - workers use public IP addresses. - encryption_spec_key_name: Customer-managed encryption key options. - If set, resources created by this pipeline will be encrypted with the - provided encryption key. Has the form: - ``projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key``. - The key needs to be in the same region as where the compute resource is - created. - - force_runner_mode: Flag to choose Beam runner. Valid options are - ``DirectRunner`` and ``Dataflow``. + predictions_format: The file format for the batch prediction results. `jsonl`, `csv`, and `bigquery` are the allowed formats, from Vertex Batch Prediction. + predictions_gcs_source: An artifact with its URI pointing toward a GCS directory with prediction or explanation files to be used for this evaluation. For prediction results, the files should be named "prediction.results-*". For explanation results, the files should be named "explanation.results-*". + predictions_bigquery_source: BigQuery table with prediction or explanation data to be used for this evaluation. For prediction results, the table column should be named "predicted_*". + ground_truth_format: Required for custom tabular and non tabular data. The file format for the ground truth files. `jsonl`, `csv`, and `bigquery` are the allowed formats. + ground_truth_gcs_source: Required for custom tabular and non tabular data. The GCS URIs representing where the ground truth is located. Used to provide ground truth for each prediction instance when they are not part of the batch prediction jobs prediction instance. + ground_truth_bigquery_source: Required for custom tabular. The BigQuery table URI representing where the ground truth is located. Used to provide ground truth for each prediction instance when they are not part of the batch prediction jobs prediction instance. + target_field_name: The target field's name. Formatted to be able to find nested columns, delimited by `.`. Prefixed with 'instance.' on the component for Vertex Batch Prediction. + model: The Vertex model used for evaluation. Must be located in the same region as the location argument. It is used to set the default configurations for AutoML and custom-trained models. + prediction_score_column: The column name of the field containing batch prediction scores. Formatted to be able to find nested columns, delimited by `.`. + dataflow_service_account: Service account to run the Dataflow job. If not set, Dataflow will use the default worker service account. For more details, see https://cloud.google.com/dataflow/docs/concepts/secURIty-and-permissions#default_worker_service_account + dataflow_disk_size_gb: The disk size (in GB) of the machine executing the evaluation run. + dataflow_machine_type: The machine type executing the evaluation run. + dataflow_workers_num: The number of workers executing the evaluation run. + dataflow_max_workers_num: The max number of workers executing the evaluation run. + dataflow_subnetwork: Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be used. More details: https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications + dataflow_use_public_ips: Specifies whether Dataflow workers use public IP addresses. + encryption_spec_key_name: Customer-managed encryption key options. If set, resources created by this pipeline will be encrypted with the provided encryption key. Has the form: `projects/my-project/locations/my-location/keyRings/my-kr/cryptoKeys/my-key`. The key needs to be in the same region as where the compute resource is created. + force_runner_mode: Flag to choose Beam runner. Valid options are `DirectRunner` and `Dataflow`. + project: Project to run evaluation container. Defaults to the project in which the PipelineJob is run. Returns: - evaluation_metrics: - ``google.RegressionMetrics`` representing the regression - evaluation metrics in GCS. - gcp_resources: Serialized gcp_resources proto tracking the Dataflow - job. For more details, see - https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. + evaluation_metrics: `google.RegressionMetrics` representing the regression evaluation metrics in GCS. + gcp_resources: Serialized gcp_resources proto tracking the Dataflow job. For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md. """ # fmt: on return dsl.ContainerSpec( - image=EVAL_IMAGE_TAG, + image=version.EVAL_IMAGE_TAG, command=[ 'python3', '/main.py', diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/vertex_notification_email/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/vertex_notification_email/component.py index 49f997e81f..6c8a35f2a9 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/vertex_notification_email/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/vertex_notification_email/component.py @@ -28,11 +28,7 @@ def vertex_pipelines_notification_email( # fmt: off """Send notification email(s) when an upstream task/DAG completes. - This component can only be used as an `ExitHandler `_'s exit task. Note that the `PipelineTaskFinalStatus `_ is provided automatically by Vertex Pipelines at runtime. You should not provide any input to this parameter when you instantiate this component as a task. - - This component works only on Vertex Pipelines. This component raises an exception when run on Kubeflow Pipelines. - - See a `usage example `_. + This component can only be used as an [ExitHandler](https://www.kubeflow.org/docs/components/pipelines/v2/pipelines/control-flow/#exit-handling-dslexithandler)'s exit task. Note that the [PipelineTaskFinalStatus](https://kubeflow-pipelines.readthedocs.io/en/latest/source/dsl.html#kfp.dsl.PipelineTaskFinalStatus) is provided automatically by Vertex Pipelines at runtime. You should not provide any input to this parameter when you instantiate this component as a task. This component works only on Vertex Pipelines. This component raises an exception when run on Kubeflow Pipelines. See a [usage example](https://cloud.google.com/vertex-ai/docs/pipelines/email-notifications). Args: recipients: A list of email addresses to send a notification to. diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/__init__.py b/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/__init__.py index 851fcc2c24..6bc60df9ef 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/__init__.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# fmt: off """Wait on the completion of GCP resources spawned from an upstream pipeline component.""" +# fmt: on from google_cloud_pipeline_components.v1.wait_gcp_resources.component import wait_gcp_resources as WaitGcpResourcesOp diff --git a/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/component.py b/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/component.py index 8c7e4f8e84..4a8647d8d3 100644 --- a/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/component.py +++ b/components/google-cloud/google_cloud_pipeline_components/v1/wait_gcp_resources/component.py @@ -24,31 +24,20 @@ def wait_gcp_resources( output__gcp_resources: OutputPath(str), ): # fmt: off - """Waits for the completion of one or more GCP resources by polling for completion statuses. + """Waits for the completion of one or more GCP resources by polling for + completion statuses. - Currently this component only supports waiting on a `DataflowJob `_ resource. + Currently this component only supports waiting on a [DataflowJob](https://cloud.google.com/config-connector/docs/reference/resource-docs/dataflow/dataflowjob) resource. To use this component, first create a component that outputs a `gcp_resources` proto as JSON, then pass it to this component's `gcp_resources` parameter. See [details](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/google_cloud_pipeline_components/proto) on how to create a `gcp_resources` proto as a component output. - To use this component, first create a component that outputs a ``gcp_resources`` proto as JSON, then pass it to this component's ``gcp_resources`` parameter. - - See `details `_ on how to create a ``gcp_resources`` proto as a component output. - - Examples: - :: - - dataflow_python_op = gcpc.v1.dataflow.LaunchPythonOp( - python_file_path=... - ) - - dataflow_wait_op = WaitGcpResourcesOp( - gcp_resources=dataflow_python_op.outputs["gcp_resources"] - ) + ``` + dataflow_python_op = gcpc.v1.dataflow.LaunchPythonOp( python_file_path=... ) dataflow_wait_op = WaitGcpResourcesOp( gcp_resources=dataflow_python_op.outputs["gcp_resources"] ) + ``` Args: - gcp_resources: Serialized JSON of ``gcp_resources`` proto, indicating the resource(s) this component should wait on. + gcp_resources: Serialized JSON of `gcp_resources` proto, indicating the resource(s) this component should wait on. Returns: - gcp_resources: The ``gcp_resource``, including any relevant error information. - + gcp_resources: The `gcp_resource`, including any relevant error information. """ # fmt: on return dsl.ContainerSpec( diff --git a/components/google-cloud/google_cloud_pipeline_components/version.py b/components/google-cloud/google_cloud_pipeline_components/version.py index 3e9709ecb4..ee5ff1441f 100644 --- a/components/google-cloud/google_cloud_pipeline_components/version.py +++ b/components/google-cloud/google_cloud_pipeline_components/version.py @@ -13,4 +13,4 @@ # limitations under the License. """Google Cloud Pipeline Components version.""" -__version__ = "2.0.0" +__version__ = "2.5.0" diff --git a/components/google-cloud/setup.cfg b/components/google-cloud/setup.cfg deleted file mode 100644 index 224a77957f..0000000000 --- a/components/google-cloud/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md \ No newline at end of file diff --git a/components/google-cloud/setup.py b/components/google-cloud/setup.py index ab6134e892..aa4b5d1c9e 100644 --- a/components/google-cloud/setup.py +++ b/components/google-cloud/setup.py @@ -18,8 +18,7 @@ import os import types -from setuptools import find_packages -from setuptools import setup +import setuptools relative_directory = os.path.relpath(os.path.dirname(os.path.abspath(__file__))) GCPC_DIR_NAME = "google_cloud_pipeline_components" @@ -36,7 +35,11 @@ with open("README.md") as fp: _GCPC_LONG_DESCRIPTION = fp.read() -setup( +yaml_data = glob.glob(relative_data_path + "/**/*.yaml", recursive=True) +json_data = glob.glob( + relative_data_path + "/**/automl/**/*.json", recursive=True +) +setuptools.setup( name="google-cloud-pipeline-components", version=version.__version__, description=( @@ -64,6 +67,7 @@ "protobuf<4.0.0dev,>=3.19.0", "grpcio-status<=1.47.0", ] + [ + "commonmark==0.9.1", "autodocsumm==0.2.9", "sphinx==5.0.2", "sphinx-immaterial==0.9.0", @@ -78,8 +82,9 @@ # Pin google-api-core version for the bug fixing in 1.31.5 # https://github.com/googleapis/python-api-core/releases/tag/v1.31.5 "google-api-core>=1.31.5,<3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0", - "kfp>=2.0.0b10,<3.0.0", + "kfp>=2.0.0b10,<=2.3.0", "google-cloud-aiplatform>=1.14.0,<2", + "Jinja2==3.1.2", ], project_urls={ "User Documentation": "https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction", @@ -87,7 +92,6 @@ "https://google-cloud-pipeline-components.readthedocs.io/" ), "Source": "https://github.com/kubeflow/pipelines/tree/master/components/google-cloud", - # TODO: update to point to reference documentation release notes once available post GCPC v2 GA "Release Notes": "https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/RELEASE.md", }, dependency_links=[], @@ -114,16 +118,11 @@ package_dir={ GCPC_DIR_NAME: os.path.join(relative_directory, GCPC_DIR_NAME) }, - packages=find_packages(where=relative_directory, include="*"), + packages=setuptools.find_packages(where=relative_directory, include="*"), package_data={ GCPC_DIR_NAME: [ x.replace(relative_data_path + "/", "") - for x in glob.glob( - relative_data_path + "/**/*.yaml", recursive=True - ) - + glob.glob( - relative_data_path + "/**/automl/**/*.json", recursive=True - ) + for x in yaml_data + json_data ] }, ) diff --git a/components/google-cloud/tox.ini b/components/google-cloud/tox.ini deleted file mode 100644 index 8183b85387..0000000000 --- a/components/google-cloud/tox.ini +++ /dev/null @@ -1,30 +0,0 @@ -[tox] -envlist = clean,py39 -skip_missing_interpreters = true - -[testenv] -usedevelop = True -install_command = pip install -U {opts} {packages} -extras = tests -testpaths = tests -setenv = - TEST_UNDECLARED_OUTPUTS_DIR = . - -deps = - pytest - pytest-cov - absl-py - -depends = - {py39}: clean - report: py39 -commands = - py.test --cov=google_cloud_pipeline_components --cov-append --cov-report=term-missing -vvv -s {posargs} - -[coverage:report] -skip_empty = true - -[testenv:clean] -deps = coverage -skip_install = true -commands = coverage erase diff --git a/components/kserve/OWNERS b/components/kserve/OWNERS index 39b3c59a1c..41cc4556df 100644 --- a/components/kserve/OWNERS +++ b/components/kserve/OWNERS @@ -1,7 +1,6 @@ approvers: - - animeshsingh - Tomcli + - yhwang reviewers: - - animeshsingh - Tomcli - yhwang diff --git a/components/kserve/README.md b/components/kserve/README.md index 7ada670447..c6a42842ef 100644 --- a/components/kserve/README.md +++ b/components/kserve/README.md @@ -4,7 +4,7 @@ Organization: KServe Organization Description: KServe is a highly scalable and standards based Model Inference Platform on Kubernetes for Trusted AI -Version information: KServe 0.10.1. Works for Kubeflow 1.7 +Version information: KServe 0.11.1. Works for Kubeflow 1.8 **Note:** To use the KServe 0.7.0 version of this component which runs on Kubeflow 1.5, then change the load_component_from_url in the usage section with the following YAML instead: ``` @@ -14,7 +14,8 @@ https://raw.githubusercontent.com/kubeflow/pipelines/1.8.1/components/kserve/com Test status: Currently manual tests Owners information: - - Animesh Singh (animeshsingh) - IBM, singhan@us.ibm.com + - Tommy Li (Tomcli) - IBM, tommy.chaoping.li@ibm.com + - Yi-Hong Wang (yhwang) - IBM, yh.wang@ibm.com ## Usage @@ -38,6 +39,9 @@ kserve_op = components.load_component_from_url('https://raw.githubusercontent.co | canary_traffic_percent | `100` | The traffic split percentage between the candidate model and the last ready model | | namespace | | Kubernetes namespace where the KServe service is deployed. If no namespace is provided, `anonymous` will be used unless a namespace is provided in the `inferenceservice_yaml` argument. | | framework | | Machine learning framework for model serving. Currently the supported frameworks are `tensorflow`, `pytorch`, `sklearn`, `xgboost`, `onnx`, `triton`, `pmml`, and `lightgbm`. | +| runtime_version | `latest` | Runtime Version of Machine Learning Framework | +| resource_requests | `{"cpu": "0.5", "memory": "512Mi"}` | CPU and Memory requests for Model Serving | +| resource_limits | `{"cpu": "1", "memory": "1Gi"}` | CPU and Memory limits for Model Serving | | custom_model_spec | `{}` | Custom model runtime container spec in JSON. Sample spec: `{"image": "codait/max-object-detector", "port":5000, "name": "test-container"}` | | inferenceservice_yaml | `{}` | Raw InferenceService serialized YAML for deployment. Use this if you need additional configurations for your InferenceService. | | autoscaling_target | `0` | Autoscaling Target Number. If not 0, sets the following annotation on the InferenceService: `autoscaling.knative.dev/target` | @@ -184,4 +188,3 @@ kserve_op( inferenceservice_yaml=isvc_yaml ) ``` - diff --git a/components/kserve/component.yaml b/components/kserve/component.yaml index 2a352d96f8..9d7b97e3e2 100644 --- a/components/kserve/component.yaml +++ b/components/kserve/component.yaml @@ -1,28 +1,31 @@ name: Serve a model with KServe description: Serve Models using KServe inputs: - - {name: Action, type: String, default: 'create', description: 'Action to execute on KServe'} - - {name: Model Name, type: String, default: '', description: 'Name to give to the deployed model'} - - {name: Model URI, type: String, default: '', description: 'Path of the S3 or GCS compatible directory containing the model.'} - - {name: Canary Traffic Percent, type: String, default: '100', description: 'The traffic split percentage between the candidate model and the last ready model'} - - {name: Namespace, type: String, default: '', description: 'Kubernetes namespace where the KServe service is deployed.'} - - {name: Framework, type: String, default: '', description: 'Machine Learning Framework for Model Serving.'} - - {name: Custom Model Spec, type: String, default: '{}', description: 'Custom model runtime container spec in JSON'} - - {name: Autoscaling Target, type: String, default: '0', description: 'Autoscaling Target Number'} - - {name: Service Account, type: String, default: '', description: 'ServiceAccount to use to run the InferenceService pod'} - - {name: Enable Istio Sidecar, type: Bool, default: 'True', description: 'Whether to enable istio sidecar injection'} - - {name: InferenceService YAML, type: String, default: '{}', description: 'Raw InferenceService serialized YAML for deployment'} - - {name: Watch Timeout, type: String, default: '300', description: "Timeout seconds for watching until InferenceService becomes ready."} - - {name: Min Replicas, type: String, default: '-1', description: 'Minimum number of InferenceService replicas'} - - {name: Max Replicas, type: String, default: '-1', description: 'Maximum number of InferenceService replicas'} - - {name: Request Timeout, type: String, default: '60', description: "Specifies the number of seconds to wait before timing out a request to the component."} - - {name: Enable ISVC Status, type: Bool, default: 'True', description: "Specifies whether to store the inference service status as the output parameter"} + - {name: Action, type: String, default: 'create', description: 'Action to execute on KServe'} + - {name: Model Name, type: String, default: '', description: 'Name to give to the deployed model'} + - {name: Model URI, type: String, default: '', description: 'Path of the S3 or GCS compatible directory containing the model.'} + - {name: Canary Traffic Percent, type: String, default: '100', description: 'The traffic split percentage between the candidate model and the last ready model'} + - {name: Namespace, type: String, default: '', description: 'Kubernetes namespace where the KServe service is deployed.'} + - {name: Framework, type: String, default: '', description: 'Machine Learning Framework for Model Serving.'} + - {name: Runtime Version, type: String, default: 'latest', description: 'Runtime Version of Machine Learning Framework'} + - {name: Resource Requests, type: String, default: '{"cpu": "0.5", "memory": "512Mi"}', description: 'CPU and Memory requests for Model Serving'} + - {name: Resource Limits, type: String, default: '{"cpu": "1", "memory": "1Gi"}', description: 'CPU and Memory limits for Model Serving'} + - {name: Custom Model Spec, type: String, default: '{}', description: 'Custom model runtime container spec in JSON'} + - {name: Autoscaling Target, type: String, default: '0', description: 'Autoscaling Target Number'} + - {name: Service Account, type: String, default: '', description: 'ServiceAccount to use to run the InferenceService pod'} + - {name: Enable Istio Sidecar, type: Bool, default: 'True', description: 'Whether to enable istio sidecar injection'} + - {name: InferenceService YAML, type: String, default: '{}', description: 'Raw InferenceService serialized YAML for deployment'} + - {name: Watch Timeout, type: String, default: '300', description: "Timeout seconds for watching until InferenceService becomes ready."} + - {name: Min Replicas, type: String, default: '-1', description: 'Minimum number of InferenceService replicas'} + - {name: Max Replicas, type: String, default: '-1', description: 'Maximum number of InferenceService replicas'} + - {name: Request Timeout, type: String, default: '60', description: "Specifies the number of seconds to wait before timing out a request to the component."} + - {name: Enable ISVC Status, type: Bool, default: 'True', description: "Specifies whether to store the inference service status as the output parameter"} outputs: - - {name: InferenceService Status, type: String, description: 'Status JSON output of InferenceService'} + - {name: InferenceService Status, type: String, description: 'Status JSON output of InferenceService'} implementation: container: - image: quay.io/aipipeline/kserve-component:v0.10.1 + image: quay.io/aipipeline/kserve-component:v0.11.1 command: ['python'] args: [ -u, kservedeployer.py, @@ -32,6 +35,9 @@ implementation: --canary-traffic-percent, {inputValue: Canary Traffic Percent}, --namespace, {inputValue: Namespace}, --framework, {inputValue: Framework}, + --runtime-version, {inputValue: Runtime Version}, + --resource-requests, {inputValue: Resource Requests}, + --resource-limits, {inputValue: Resource Limits}, --custom-model-spec, {inputValue: Custom Model Spec}, --autoscaling-target, {inputValue: Autoscaling Target}, --service-account, {inputValue: Service Account}, diff --git a/components/kserve/requirements.txt b/components/kserve/requirements.txt index 2be51060b8..bdab9d8f8a 100644 --- a/components/kserve/requirements.txt +++ b/components/kserve/requirements.txt @@ -1,2 +1,2 @@ -kserve==0.10.1 +kserve==0.11.1 protobuf~=3.19.0 diff --git a/components/kserve/src/kservedeployer.py b/components/kserve/src/kservedeployer.py index db84e41727..c8799332f7 100644 --- a/components/kserve/src/kservedeployer.py +++ b/components/kserve/src/kservedeployer.py @@ -21,6 +21,7 @@ import yaml from kubernetes import client +from kubernetes.client.models import V1ResourceRequirements from kserve import constants from kserve import KServeClient @@ -50,8 +51,9 @@ } -def create_predictor_spec(framework, storage_uri, canary_traffic_percent, - service_account, min_replicas, max_replicas, containers, request_timeout): +def create_predictor_spec(framework, runtime_version, resource_requests, resource_limits, + storage_uri, canary_traffic_percent, service_account, min_replicas, + max_replicas, containers, request_timeout): """ Create and return V1beta1PredictorSpec to be used in a V1beta1InferenceServiceSpec object. @@ -81,7 +83,14 @@ def create_predictor_spec(framework, storage_uri, canary_traffic_percent, setattr( predictor_spec, framework, - AVAILABLE_FRAMEWORKS[framework](storage_uri=storage_uri) + AVAILABLE_FRAMEWORKS[framework]( + storage_uri=storage_uri, + resources=V1ResourceRequirements( + requests=resource_requests, + limits=resource_limits + ), + runtime_version=runtime_version + ) ) return predictor_spec @@ -178,10 +187,10 @@ def submit_api_request(kserve_client, action, name, isvc, namespace=None, return outputs -def perform_action(action, model_name, model_uri, canary_traffic_percent, namespace, - framework, custom_model_spec, service_account, inferenceservice_yaml, - request_timeout, autoscaling_target=0, enable_istio_sidecar=True, - watch_timeout=300, min_replicas=0, max_replicas=0): +def perform_action(action, model_name, model_uri, canary_traffic_percent, namespace, framework, + runtime_version, resource_requests, resource_limits, custom_model_spec, + service_account, inferenceservice_yaml, request_timeout, autoscaling_target=0, + enable_istio_sidecar=True, watch_timeout=300, min_replicas=0, max_replicas=0): """ Perform the specified action. If the action is not 'delete' and `inferenceService_yaml` was provided, the dict representation of the YAML will be sent directly to the @@ -224,8 +233,9 @@ def perform_action(action, model_name, model_uri, canary_traffic_percent, namesp # Build the V1beta1PredictorSpec. predictor_spec = create_predictor_spec( - framework, model_uri, canary_traffic_percent, service_account, - min_replicas, max_replicas, containers, request_timeout + framework, runtime_version, resource_requests, resource_limits, + model_uri, canary_traffic_percent, service_account, min_replicas, + max_replicas, containers, request_timeout ) isvc = create_inference_service(metadata, predictor_spec) @@ -287,6 +297,24 @@ def main(): str(list(AVAILABLE_FRAMEWORKS.keys())), default="" ) + parser.add_argument( + "--runtime-version", + type=str, + help="Runtime Version of Machine Learning Framework", + default="latest" + ) + parser.add_argument( + "--resource-requests", + type=json.loads, + help="CPU and Memory requests for Model Serving", + default='{"cpu": "0.5", "memory": "512Mi"}', + ) + parser.add_argument( + "--resource-limits", + type=json.loads, + help="CPU and Memory limits for Model Serving", + default='{"cpu": "1", "memory": "1Gi"}', + ) parser.add_argument( "--custom-model-spec", type=json.loads, @@ -342,6 +370,9 @@ def main(): canary_traffic_percent = int(args.canary_traffic_percent) namespace = args.namespace framework = args.framework.lower() + runtime_version = args.runtime_version.lower() + resource_requests = args.resource_requests + resource_limits = args.resource_limits output_path = args.output_path custom_model_spec = args.custom_model_spec autoscaling_target = int(args.autoscaling_target) @@ -381,6 +412,9 @@ def main(): canary_traffic_percent=canary_traffic_percent, namespace=namespace, framework=framework, + runtime_version=runtime_version, + resource_requests=resource_requests, + resource_limits=resource_limits, custom_model_spec=custom_model_spec, autoscaling_target=autoscaling_target, service_account=service_account, diff --git a/docs/OWNERS b/docs/OWNERS index 7b6c1ed48a..bbea284a9d 100644 --- a/docs/OWNERS +++ b/docs/OWNERS @@ -1,8 +1,6 @@ approvers: - - Ark-kun - - gaoning777 - - hongye-sun + - chensun + - connor-mccarthy reviewers: - - Ark-kun - - gaoning777 - - hongye-sun \ No newline at end of file + - chensun + - connor-mccarthy diff --git a/docs/build_docs_locally.sh b/docs/build_docs_locally.sh new file mode 100644 index 0000000000..74a31608e9 --- /dev/null +++ b/docs/build_docs_locally.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2023 Kubeflow Pipelines contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# install requirements + +pushd .. +pip install -r docs/requirements.txt +popd + +# build docs +make clean html + +# serve docs +pushd _build/html +python3 -m http.server +popd diff --git a/docs/conf.py b/docs/conf.py index 42171b3f67..253fae4000 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -132,12 +132,26 @@ True, 'version_info': [ # need to use the sdk- prefix to avoid conflict with the BE's GitHub release tags + { + 'version': + 'https://kubeflow-pipelines.readthedocs.io/en/sdk-2.3.0/', + 'title': + '2.3.0', + 'aliases': ['stable'], + }, + { + 'version': + 'https://kubeflow-pipelines.readthedocs.io/en/sdk-2.2.0/', + 'title': + '2.2.0', + 'aliases': [], + }, { 'version': 'https://kubeflow-pipelines.readthedocs.io/en/sdk-2.0.1/', 'title': '2.0.1', - 'aliases': ['stable'], + 'aliases': [], }, { 'version': diff --git a/frontend/.nvmrc b/frontend/.nvmrc index cae54a258e..4a9c19cb52 100644 --- a/frontend/.nvmrc +++ b/frontend/.nvmrc @@ -1 +1 @@ -v12.14.1 +v14.21.3 diff --git a/frontend/README.md b/frontend/README.md index 6880f3e72e..7a3766f01d 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -176,8 +176,22 @@ If a file in [pipelines/third_party/ml-metadata/ml_metadata/proto](third_party/m * Add `protoc` ([download](https://github.com/protocolbuffers/protobuf/releases)) to your system PATH + + ```bash + # Example: + apt install -y protobuf-compiler=3.15.8 + ``` + * Add `protoc-gen-grpc-web` ([download](https://github.com/grpc/grpc-web/releases)) to your system PATH + + ```bash + # Example: + curl -LO https://github.com/grpc/grpc-web/releases/download/1.4.2/protoc-gen-grpc-web-1.4.2-linux-x86_64 + mv protoc-gen-grpc-web-1.4.2-linux-x86_64 /usr/local/bin/protoc-gen-grpc-web + chmod +x /usr/local/bin/protoc-gen-grpc-web + ``` + * Replace `metadata_store.proto` and `metadata_store_service.proto` proto files with target mlmd version by running ```bash diff --git a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json index 7a2f6abe75..2e5c699128 100644 --- a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json +++ b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline.json @@ -107,7 +107,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nfrom kfp.dsl import *\nfrom typing import *\n\ndef preprocess(\n # An input parameter of type string.\n message: str,\n # An input parameter of type dict.\n input_dict_parameter: Dict[str, int],\n # An input parameter of type list.\n input_list_parameter: List[str],\n # Use Output[T] to get a metadata-rich handle to the output artifact\n # of type `Dataset`.\n output_dataset_one: Output[Dataset],\n # A locally accessible filepath for another output artifact of type\n # `Dataset`.\n output_dataset_two_path: OutputPath('Dataset'),\n # A locally accessible filepath for an output parameter of type string.\n output_parameter_path: OutputPath(str),\n # A locally accessible filepath for an output parameter of type bool.\n output_bool_parameter_path: OutputPath(bool),\n # A locally accessible filepath for an output parameter of type dict.\n output_dict_parameter_path: OutputPath(Dict[str, int]),\n # A locally accessible filepath for an output parameter of type list.\n output_list_parameter_path: OutputPath(List[str]),\n):\n \"\"\"Dummy preprocessing step.\"\"\"\n\n # Use Dataset.path to access a local file path for writing.\n # One can also use Dataset.uri to access the actual URI file path.\n with open(output_dataset_one.path, 'w') as f:\n f.write(message)\n\n # OutputPath is used to just pass the local file path of the output artifact\n # to the function.\n with open(output_dataset_two_path, 'w') as f:\n f.write(message)\n\n with open(output_parameter_path, 'w') as f:\n f.write(message)\n\n with open(output_bool_parameter_path, 'w') as f:\n f.write(\n str(True)) # use either `str()` or `json.dumps()` for bool values.\n\n import json\n with open(output_dict_parameter_path, 'w') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with open(output_list_parameter_path, 'w') as f:\n f.write(json.dumps(input_list_parameter))\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-train": { @@ -122,7 +122,7 @@ "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "\nimport json\nimport inspect\nfrom typing import *\n\n# Copyright 2021 The Kubeflow Authors\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\"\"\"Classes for input/output types in KFP SDK.\n\nThese are only compatible with v2 Pipelines.\n\"\"\"\n\nimport os\nfrom typing import Dict, Generic, List, Optional, Type, TypeVar, Union\n\n\n_GCS_LOCAL_MOUNT_PREFIX = '/gcs/'\n_MINIO_LOCAL_MOUNT_PREFIX = '/minio/'\n_S3_LOCAL_MOUNT_PREFIX = '/s3/'\n\n\nclass Artifact(object):\n \"\"\"Generic Artifact class.\n\n This class is meant to represent the metadata around an input or output\n machine-learning Artifact. Artifacts have URIs, which can either be a location\n on disk (or Cloud storage) or some other resource identifier such as\n an API resource name.\n\n Artifacts carry a `metadata` field, which is a dictionary for storing\n metadata related to this artifact.\n \"\"\"\n TYPE_NAME = 'system.Artifact'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n \"\"\"Initializes the Artifact with the given name, URI and metadata.\"\"\"\n self.uri = uri or ''\n self.name = name or ''\n self.metadata = metadata or {}\n\n @property\n def path(self):\n return self._get_path()\n\n @path.setter\n def path(self, path):\n self._set_path(path)\n\n def _get_path(self) -> Optional[str]:\n if self.uri.startswith('gs://'):\n return _GCS_LOCAL_MOUNT_PREFIX + self.uri[len('gs://'):]\n elif self.uri.startswith('minio://'):\n return _MINIO_LOCAL_MOUNT_PREFIX + self.uri[len('minio://'):]\n elif self.uri.startswith('s3://'):\n return _S3_LOCAL_MOUNT_PREFIX + self.uri[len('s3://'):]\n return None\n\n def _set_path(self, path):\n if path.startswith(_GCS_LOCAL_MOUNT_PREFIX):\n path = 'gs://' + path[len(_GCS_LOCAL_MOUNT_PREFIX):]\n elif path.startswith(_MINIO_LOCAL_MOUNT_PREFIX):\n path = 'minio://' + path[len(_MINIO_LOCAL_MOUNT_PREFIX):]\n elif path.startswith(_S3_LOCAL_MOUNT_PREFIX):\n path = 's3://' + path[len(_S3_LOCAL_MOUNT_PREFIX):]\n self.uri = path\n\n\nclass Model(Artifact):\n \"\"\"An artifact representing an ML Model.\"\"\"\n TYPE_NAME = 'system.Model'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n @property\n def framework(self) -> str:\n return self._get_framework()\n\n def _get_framework(self) -> str:\n return self.metadata.get('framework', '')\n\n @framework.setter\n def framework(self, framework: str):\n self._set_framework(framework)\n\n def _set_framework(self, framework: str):\n self.metadata['framework'] = framework\n\n\nclass Dataset(Artifact):\n \"\"\"An artifact representing an ML Dataset.\"\"\"\n TYPE_NAME = 'system.Dataset'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n\nclass Metrics(Artifact):\n \"\"\"Represent a simple base Artifact type to store key-value scalar metrics.\"\"\"\n TYPE_NAME = 'system.Metrics'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n def log_metric(self, metric: str, value: float):\n \"\"\"Sets a custom scalar metric.\n\n Args:\n metric: Metric key\n value: Value of the metric.\n \"\"\"\n self.metadata[metric] = value\n\n\nclass ClassificationMetrics(Artifact):\n \"\"\"Represents Artifact class to store Classification Metrics.\"\"\"\n TYPE_NAME = 'system.ClassificationMetrics'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n def log_roc_data_point(self, fpr: float, tpr: float, threshold: float):\n \"\"\"Logs a single data point in the ROC Curve.\n\n Args:\n fpr: False positive rate value of the data point.\n tpr: True positive rate value of the data point.\n threshold: Threshold value for the data point.\n \"\"\"\n\n roc_reading = {\n 'confidenceThreshold': threshold,\n 'recall': tpr,\n 'falsePositiveRate': fpr\n }\n if 'confidenceMetrics' not in self.metadata.keys():\n self.metadata['confidenceMetrics'] = []\n\n self.metadata['confidenceMetrics'].append(roc_reading)\n\n def log_roc_curve(self, fpr: List[float], tpr: List[float],\n threshold: List[float]):\n \"\"\"Logs an ROC curve.\n\n The list length of fpr, tpr and threshold must be the same.\n\n Args:\n fpr: List of false positive rate values.\n tpr: List of true positive rate values.\n threshold: List of threshold values.\n \"\"\"\n if len(fpr) != len(tpr) or len(fpr) != len(threshold) or len(tpr) != len(\n threshold):\n raise ValueError('Length of fpr, tpr and threshold must be the same. '\n 'Got lengths {}, {} and {} respectively.'.format(\n len(fpr), len(tpr), len(threshold)))\n\n for i in range(len(fpr)):\n self.log_roc_data_point(fpr=fpr[i], tpr=tpr[i], threshold=threshold[i])\n\n def set_confusion_matrix_categories(self, categories: List[str]):\n \"\"\"Stores confusion matrix categories.\n\n Args:\n categories: List of strings specifying the categories.\n \"\"\"\n\n self._categories = []\n annotation_specs = []\n for category in categories:\n annotation_spec = {'displayName': category}\n self._categories.append(category)\n annotation_specs.append(annotation_spec)\n\n self._matrix = []\n for row in range(len(self._categories)):\n self._matrix.append({'row': [0] * len(self._categories)})\n\n self._confusion_matrix = {}\n self._confusion_matrix['annotationSpecs'] = annotation_specs\n self._confusion_matrix['rows'] = self._matrix\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n def log_confusion_matrix_row(self, row_category: str, row: List[float]):\n \"\"\"Logs a confusion matrix row.\n\n Args:\n row_category: Category to which the row belongs.\n row: List of integers specifying the values for the row.\n\n Raises:\n ValueError: If row_category is not in the list of categories\n set in set_categories call.\n \"\"\"\n if row_category not in self._categories:\n raise ValueError('Invalid category: {} passed. Expected one of: {}'.\\\n format(row_category, self._categories))\n\n if len(row) != len(self._categories):\n raise ValueError('Invalid row. Expected size: {} got: {}'.\\\n format(len(self._categories), len(row)))\n\n self._matrix[self._categories.index(row_category)] = {'row': row}\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n def log_confusion_matrix_cell(self, row_category: str, col_category: str,\n value: int):\n \"\"\"Logs a cell in the confusion matrix.\n\n Args:\n row_category: String representing the name of the row category.\n col_category: String representing the name of the column category.\n value: Int value of the cell.\n\n Raises:\n ValueError: If row_category or col_category is not in the list of\n categories set in set_categories.\n \"\"\"\n if row_category not in self._categories:\n raise ValueError('Invalid category: {} passed. Expected one of: {}'.\\\n format(row_category, self._categories))\n\n if col_category not in self._categories:\n raise ValueError('Invalid category: {} passed. Expected one of: {}'.\\\n format(row_category, self._categories))\n\n self._matrix[self._categories.index(row_category)]['row'][\n self._categories.index(col_category)] = value\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n def log_confusion_matrix(self, categories: List[str],\n matrix: List[List[int]]):\n \"\"\"Logs a confusion matrix.\n\n Args:\n categories: List of the category names.\n matrix: Complete confusion matrix.\n\n Raises:\n ValueError: Length of categories does not match number of rows or columns.\n \"\"\"\n self.set_confusion_matrix_categories(categories)\n\n if len(matrix) != len(categories):\n raise ValueError('Invalid matrix: {} passed for categories: {}'.\\\n format(matrix, categories))\n\n for index in range(len(categories)):\n if len(matrix[index]) != len(categories):\n raise ValueError('Invalid matrix: {} passed for categories: {}'.\\\n format(matrix, categories))\n\n self.log_confusion_matrix_row(categories[index], matrix[index])\n\n self.metadata['confusionMatrix'] = self._confusion_matrix\n\n\nclass SlicedClassificationMetrics(Artifact):\n \"\"\"Metrics class representing Sliced Classification Metrics.\n\n Similar to ClassificationMetrics clients using this class are expected to use\n log methods of the class to log metrics with the difference being each log\n method takes a slice to associate the ClassificationMetrics.\n\n \"\"\"\n\n TYPE_NAME = 'system.SlicedClassificationMetrics'\n\n def __init__(self,\n name: Optional[str] = None,\n uri: Optional[str] = None,\n metadata: Optional[Dict] = None):\n super().__init__(uri=uri, name=name, metadata=metadata)\n\n def _upsert_classification_metrics_for_slice(self, slice: str):\n \"\"\"Upserts the classification metrics instance for a slice.\"\"\"\n if slice not in self._sliced_metrics:\n self._sliced_metrics[slice] = ClassificationMetrics()\n\n def _update_metadata(self, slice: str):\n \"\"\"Updates metadata to adhere to the metrics schema.\"\"\"\n self.metadata = {}\n self.metadata['evaluationSlices'] = []\n for slice in self._sliced_metrics.keys():\n slice_metrics = {\n 'slice': slice,\n 'sliceClassificationMetrics': self._sliced_metrics[slice].metadata\n }\n self.metadata['evaluationSlices'].append(slice_metrics)\n\n def log_roc_reading(self, slice: str, threshold: float, tpr: float,\n fpr: float):\n \"\"\"Logs a single data point in the ROC Curve of a slice.\n\n Args:\n slice: String representing slice label.\n threshold: Thresold value for the data point.\n tpr: True positive rate value of the data point.\n fpr: False positive rate value of the data point.\n \"\"\"\n\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_roc_reading(threshold, tpr, fpr)\n self._update_metadata(slice)\n\n def load_roc_readings(self, slice: str, readings: List[List[float]]):\n \"\"\"Supports bulk loading ROC Curve readings for a slice.\n\n Args:\n slice: String representing slice label.\n readings: A 2-D list providing ROC Curve data points.\n The expected order of the data points is: threshold,\n true_positive_rate, false_positive_rate.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].load_roc_readings(readings)\n self._update_metadata(slice)\n\n def set_confusion_matrix_categories(self, slice: str, categories: List[str]):\n \"\"\"Stores confusion matrix categories for a slice..\n\n Categories are stored in the internal metrics_utils.ConfusionMatrix\n instance of the slice.\n\n Args:\n slice: String representing slice label.\n categories: List of strings specifying the categories.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].set_confusion_matrix_categories(categories)\n self._update_metadata(slice)\n\n def log_confusion_matrix_row(self, slice: str, row_category: str,\n row: List[int]):\n \"\"\"Logs a confusion matrix row for a slice.\n\n Row is updated on the internal metrics_utils.ConfusionMatrix\n instance of the slice.\n\n Args:\n slice: String representing slice label.\n row_category: Category to which the row belongs.\n row: List of integers specifying the values for the row.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_confusion_matrix_row(row_category, row)\n self._update_metadata(slice)\n\n def log_confusion_matrix_cell(self, slice: str, row_category: str,\n col_category: str, value: int):\n \"\"\"Logs a confusion matrix cell for a slice..\n\n Cell is updated on the internal metrics_utils.ConfusionMatrix\n instance of the slice.\n\n Args:\n slice: String representing slice label.\n row_category: String representing the name of the row category.\n col_category: String representing the name of the column category.\n value: Int value of the cell.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_confusion_matrix_cell(\n row_category, col_category, value)\n self._update_metadata(slice)\n\n def load_confusion_matrix(self, slice: str, categories: List[str],\n matrix: List[List[int]]):\n \"\"\"Supports bulk loading the whole confusion matrix for a slice.\n\n Args:\n slice: String representing slice label.\n categories: List of the category names.\n matrix: Complete confusion matrix.\n \"\"\"\n self._upsert_classification_metrics_for_slice(slice)\n self._sliced_metrics[slice].log_confusion_matrix_cell(categories, matrix)\n self._update_metadata(slice)\n\n\nT = TypeVar('T')\n\n\nclass InputAnnotation():\n \"\"\"Marker type for input artifacts.\"\"\"\n pass\n\n\n\nclass OutputAnnotation():\n \"\"\"Marker type for output artifacts.\"\"\"\n pass\n\n\n# TODO: Use typing.Annotated instead of this hack.\n# With typing.Annotated (Python 3.9+ or typing_extensions package), the\n# following would look like:\n# Input = typing.Annotated[T, InputAnnotation]\n# Output = typing.Annotated[T, OutputAnnotation]\n\n\n# Input represents an Input artifact of type T.\nInput = Union[T, InputAnnotation]\n\n# Output represents an Output artifact of type T.\nOutput = Union[T, OutputAnnotation]\n\n\ndef is_artifact_annotation(typ) -> bool:\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n return len(subs_tree) == 3 and subs_tree[0] == Union and subs_tree[2] in [InputAnnotation, OutputAnnotation]\n\n if not hasattr(typ, '__origin__'):\n return False\n\n\n if typ.__origin__ != Union and type(typ.__origin__) != type(Union):\n return False\n\n\n if not hasattr(typ, '__args__') or len(typ.__args__) != 2:\n return False\n\n if typ.__args__[1] not in [InputAnnotation, OutputAnnotation]:\n return False\n\n return True\n\ndef is_input_artifact(typ) -> bool:\n \"\"\"Returns True if typ is of type Input[T].\"\"\"\n if not is_artifact_annotation(typ):\n return False\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n return len(subs_tree) == 3 and subs_tree[2] == InputAnnotation\n\n return typ.__args__[1] == InputAnnotation\n\ndef is_output_artifact(typ) -> bool:\n \"\"\"Returns True if typ is of type Output[T].\"\"\"\n if not is_artifact_annotation(typ):\n return False\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n return len(subs_tree) == 3 and subs_tree[2] == OutputAnnotation\n\n return typ.__args__[1] == OutputAnnotation\n\ndef get_io_artifact_class(typ):\n if not is_artifact_annotation(typ):\n return None\n if typ == Input or typ == Output:\n return None\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n if len(subs_tree) != 3:\n return None\n return subs_tree[1]\n\n return typ.__args__[0]\n\ndef get_io_artifact_annotation(typ):\n if not is_artifact_annotation(typ):\n return None\n\n if hasattr(typ, '_subs_tree'): # Python 3.6\n subs_tree = typ._subs_tree()\n if len(subs_tree) != 3:\n return None\n return subs_tree[2]\n\n return typ.__args__[1]\n\n\n\n_SCHEMA_TITLE_TO_TYPE: Dict[str, Artifact] = {\n x.TYPE_NAME: x\n for x in [Artifact, Model, Dataset, Metrics, ClassificationMetrics]\n}\n\n\ndef create_runtime_artifact(runtime_artifact: Dict) -> Artifact:\n \"\"\"Creates an Artifact instance from the specified RuntimeArtifact.\n\n Args:\n runtime_artifact: Dictionary representing JSON-encoded RuntimeArtifact.\n \"\"\"\n schema_title = runtime_artifact.get('type', {}).get('schemaTitle', '')\n\n artifact_type = _SCHEMA_TITLE_TO_TYPE.get(schema_title)\n if not artifact_type:\n artifact_type = Artifact\n return artifact_type(\n uri=runtime_artifact.get('uri', ''),\n name=runtime_artifact.get('name', ''),\n metadata=runtime_artifact.get('metadata', {}),\n )\n\nclass InputPath:\n '''When creating component from function, :class:`.InputPath` should be used as function parameter annotation to tell the system to pass the *data file path* to the function instead of passing the actual data.'''\n def __init__(self, type=None):\n self.type = type\n\nclass OutputPath:\n '''When creating component from function, :class:`.OutputPath` should be used as function parameter annotation to tell the system that the function wants to output data by writing it into a file with the given path instead of returning the data from the function.'''\n def __init__(self, type=None):\n self.type = type\n\nclass Executor():\n \"\"\"Executor executes v2-based Python function components.\"\"\"\n\n def __init__(self, executor_input: Dict, function_to_execute: Callable):\n self._func = function_to_execute\n self._input = executor_input\n self._input_artifacts: Dict[str, Artifact] = {}\n self._output_artifacts: Dict[str, Artifact] = {}\n\n for name, artifacts in self._input.get('inputs', {}).get('artifacts',\n {}).items():\n artifacts_list = artifacts.get('artifacts')\n if artifacts_list:\n self._input_artifacts[name] = self._make_input_artifact(\n artifacts_list[0])\n\n for name, artifacts in self._input.get('outputs', {}).get('artifacts',\n {}).items():\n artifacts_list = artifacts.get('artifacts')\n if artifacts_list:\n self._output_artifacts[name] = self._make_output_artifact(\n artifacts_list[0])\n\n self._return_annotation = inspect.signature(self._func).return_annotation\n self._executor_output = {}\n\n @classmethod\n def _make_input_artifact(cls, runtime_artifact: Dict):\n return create_runtime_artifact(runtime_artifact)\n\n @classmethod\n def _make_output_artifact(cls, runtime_artifact: Dict):\n import os\n artifact = create_runtime_artifact(runtime_artifact)\n os.makedirs(os.path.dirname(artifact.path), exist_ok=True)\n return artifact\n\n def _get_input_artifact(self, name: str):\n return self._input_artifacts.get(name)\n\n def _get_output_artifact(self, name: str):\n return self._output_artifacts.get(name)\n\n def _get_input_parameter_value(self, parameter_name: str, parameter_type: Any):\n parameter = self._input.get('inputs', {}).get('parameters',\n {}).get(parameter_name, None)\n if parameter is None:\n return None\n\n if parameter.get('stringValue'):\n if parameter_type == str:\n return parameter['stringValue']\n elif parameter_type == bool:\n # Use `.lower()` so it can also handle 'True' and 'False' (resulted from\n # `str(True)` and `str(False)`, respectively.\n return json.loads(parameter['stringValue'].lower())\n else:\n return json.loads(parameter['stringValue'])\n elif parameter.get('intValue'):\n return int(parameter['intValue'])\n elif parameter.get('doubleValue'):\n return float(parameter['doubleValue'])\n\n def _get_output_parameter_path(self, parameter_name: str):\n parameter_name = self._maybe_strip_path_suffix(parameter_name)\n parameter = self._input.get('outputs',\n {}).get('parameters',\n {}).get(parameter_name, None)\n if parameter is None:\n return None\n\n import os\n path = parameter.get('outputFile', None)\n if path:\n os.makedirs(os.path.dirname(path), exist_ok=True)\n return path\n\n def _get_output_artifact_path(self, artifact_name: str):\n artifact_name = self._maybe_strip_path_suffix(artifact_name)\n output_artifact = self._output_artifacts.get(artifact_name)\n if not output_artifact:\n raise ValueError(\n 'Failed to get output artifact path for artifact name {}'.format(\n artifact_name))\n return output_artifact.path\n\n def _get_input_artifact_path(self, artifact_name: str):\n artifact_name = self._maybe_strip_path_suffix(artifact_name)\n input_artifact = self._input_artifacts.get(artifact_name)\n if not input_artifact:\n raise ValueError(\n 'Failed to get input artifact path for artifact name {}'.format(\n artifact_name))\n return input_artifact.path\n\n def _write_output_parameter_value(self, name: str,\n value: Union[str, int, float, bool, dict,\n list, Dict, List]):\n if type(value) == str:\n output = {'stringValue': value}\n elif type(value) == int:\n output = {'intValue': value}\n elif type(value) == float:\n output = {'doubleValue': value}\n else:\n # For bool, list, dict, List, Dict, json serialize the value.\n output = {'stringValue': json.dumps(value)}\n\n if not self._executor_output.get('parameters'):\n self._executor_output['parameters'] = {}\n\n self._executor_output['parameters'][name] = output\n\n def _write_output_artifact_payload(self, name: str, value: Any):\n path = self._get_output_artifact_path(name)\n with open(path, 'w') as f:\n f.write(str(value))\n\n # TODO: extract to a util\n @classmethod\n def _get_short_type_name(cls, type_name: str) -> str:\n \"\"\"Extracts the short form type name.\n\n This method is used for looking up serializer for a given type.\n\n For example:\n typing.List -> List\n typing.List[int] -> List\n typing.Dict[str, str] -> Dict\n List -> List\n str -> str\n\n Args:\n type_name: The original type name.\n\n Returns:\n The short form type name or the original name if pattern doesn't match.\n \"\"\"\n import re\n match = re.match('(typing\\.)?(?P\\w+)(?:\\[.+\\])?', type_name)\n if match:\n return match.group('type')\n else:\n return type_name\n\n # TODO: merge with type_utils.is_parameter_type\n @classmethod\n def _is_parameter(cls, annotation: Any) -> bool:\n if type(annotation) == type:\n return annotation in [str, int, float, bool, dict, list]\n\n # Annotation could be, for instance `typing.Dict[str, str]`, etc.\n return cls._get_short_type_name(str(annotation)) in ['Dict', 'List']\n\n @classmethod\n def _is_artifact(cls, annotation: Any) -> bool:\n if type(annotation) == type:\n return issubclass(annotation, Artifact)\n return False\n\n @classmethod\n def _is_named_tuple(cls, annotation: Any) -> bool:\n if type(annotation) == type:\n return issubclass(annotation, tuple) and hasattr(\n annotation, '_fields') and hasattr(annotation, '__annotations__')\n return False\n\n def _handle_single_return_value(self, output_name: str, annotation_type: Any,\n return_value: Any):\n if self._is_parameter(annotation_type):\n if type(return_value) != annotation_type:\n raise ValueError(\n 'Function `{}` returned value of type {}; want type {}'.format(\n self._func.__name__, type(return_value), annotation_type))\n self._write_output_parameter_value(output_name, return_value)\n elif self._is_artifact(annotation_type):\n self._write_output_artifact_payload(output_name, return_value)\n else:\n raise RuntimeError(\n 'Unknown return type: {}. Must be one of `str`, `int`, `float`, or a'\n ' subclass of `Artifact`'.format(annotation_type))\n\n def _write_executor_output(self, func_output: Optional[Any] = None):\n if self._output_artifacts:\n self._executor_output['artifacts'] = {}\n\n for name, artifact in self._output_artifacts.items():\n runtime_artifact = {\n 'name': artifact.name,\n 'uri': artifact.uri,\n 'metadata': artifact.metadata,\n }\n artifacts_list = {'artifacts': [runtime_artifact]}\n\n self._executor_output['artifacts'][name] = artifacts_list\n\n if func_output is not None:\n if self._is_parameter(self._return_annotation) or self._is_artifact(\n self._return_annotation):\n # Note: single output is named `Output` in component.yaml.\n self._handle_single_return_value('Output', self._return_annotation,\n func_output)\n elif self._is_named_tuple(self._return_annotation):\n if len(self._return_annotation._fields) != len(func_output):\n raise RuntimeError(\n 'Expected {} return values from function `{}`, got {}'.format(\n len(self._return_annotation._fields), self._func.__name__,\n len(func_output)))\n for i in range(len(self._return_annotation._fields)):\n field = self._return_annotation._fields[i]\n field_type = self._return_annotation.__annotations__[field]\n if type(func_output) == tuple:\n field_value = func_output[i]\n else:\n field_value = getattr(func_output, field)\n self._handle_single_return_value(field, field_type, field_value)\n else:\n raise RuntimeError(\n 'Unknown return type: {}. Must be one of `str`, `int`, `float`, a'\n ' subclass of `Artifact`, or a NamedTuple collection of these types.'\n .format(self._return_annotation))\n\n import os\n os.makedirs(\n os.path.dirname(self._input['outputs']['outputFile']), exist_ok=True)\n with open(self._input['outputs']['outputFile'], 'w') as f:\n f.write(json.dumps(self._executor_output))\n\n def _maybe_strip_path_suffix(self, name) -> str:\n if name.endswith('_path'):\n name = name[0:-len('_path')]\n if name.endswith('_file'):\n name = name[0:-len('_file')]\n return name\n\n def execute(self):\n annotations = inspect.getfullargspec(self._func).annotations\n\n # Function arguments.\n func_kwargs = {}\n\n for k, v in annotations.items():\n if k == 'return':\n continue\n\n if self._is_parameter(v):\n func_kwargs[k] = self._get_input_parameter_value(k, v)\n\n if is_artifact_annotation(v):\n if is_input_artifact(v):\n func_kwargs[k] = self._get_input_artifact(k)\n if is_output_artifact(v):\n func_kwargs[k] = self._get_output_artifact(k)\n\n elif isinstance(v, OutputPath):\n if self._is_parameter(v.type):\n func_kwargs[k] = self._get_output_parameter_path(k)\n else:\n func_kwargs[k] = self._get_output_artifact_path(k)\n elif isinstance(v, InputPath):\n func_kwargs[k] = self._get_input_artifact_path(k)\n\n result = self._func(**func_kwargs)\n self._write_executor_output(result)\n\n\ndef train(\n # Use InputPath to get a locally accessible path for the input artifact\n # of type `Dataset`.\n dataset_one_path: InputPath('Dataset'),\n # Use Input[T] to get a metadata-rich handle to the input artifact\n # of type `Dataset`.\n dataset_two: Input[Dataset],\n # An input parameter of type string.\n message: str,\n # Use Output[T] to get a metadata-rich handle to the output artifact\n # of type `Dataset`.\n model: Output[Model],\n # An input parameter of type bool.\n input_bool: bool,\n # An input parameter of type dict.\n input_dict: Dict[str, int],\n # An input parameter of type List[str].\n input_list: List[str],\n # An input parameter of type int with a default value.\n num_steps: int = 100,\n):\n \"\"\"Dummy Training step\"\"\"\n with open(dataset_one_path, 'r') as input_file:\n dataset_one_contents = input_file.read()\n\n with open(dataset_two.path, 'r') as input_file:\n dataset_two_contents = input_file.read()\n\n line = (f'dataset_one_contents: {dataset_one_contents} || '\n f'dataset_two_contents: {dataset_two_contents} || '\n f'message: {message} || '\n f'input_bool: {input_bool}, type {type(input_bool)} || '\n f'input_dict: {input_dict}, type {type(input_dict)} || '\n f'input_list: {input_list}, type {type(input_list)} \\n')\n\n with open(model.path, 'w') as output_file:\n for i in range(num_steps):\n output_file.write('Step {}\\n{}\\n=====\\n'.format(i, line))\n\n # model is an instance of Model artifact, which has a .metadata dictionary\n # to store arbitrary metadata for the output artifact.\n model.metadata['accuracy'] = 0.9\n\n\ndef executor_main():\n import argparse\n import json\n\n parser = argparse.ArgumentParser(description='Process some integers.')\n parser.add_argument('--executor_input', type=str)\n parser.add_argument('--function_to_execute', type=str)\n\n args, _ = parser.parse_known_args()\n executor_input = json.loads(args.executor_input)\n function_to_execute = globals()[args.function_to_execute]\n\n executor = Executor(executor_input=executor_input,\n function_to_execute=function_to_execute)\n\n executor.execute()\n\n\nif __name__ == '__main__':\n executor_main()\n" ], - "image": "python:3.7" + "image": "python:3.9" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml index 3d074610df..58b6df51e7 100644 --- a/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml +++ b/frontend/mock-backend/data/v2/pipeline/lightweight_python_functions_v2_pipeline_rev.yaml @@ -6,7 +6,7 @@ deploymentSpec: executors: exec-preprocess: container: - image: python:3.7 + image: python:3.9 args: - --executor_input - '{{$}}' @@ -105,7 +105,7 @@ deploymentSpec: \ Model artifact, which has a .metadata dictionary\n # to store arbitrary\ \ metadata for the output artifact.\n model.metadata['accuracy'] = 0.9\n\ \n" - image: python:3.7 + image: python:3.9 components: comp-preprocess: inputDefinitions: @@ -233,4 +233,4 @@ root: enableCache: true componentRef: name: comp-preprocess -defaultPipelineRoot: dummy_root +defaultPipelineRoot: minio://dummy_root diff --git a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json index f2364e2af3..6f4f6f7a7e 100644 --- a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json +++ b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.json @@ -929,7 +929,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -> list:\n return [\n {\n 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n },\n ]\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-args-generator-op-2": { @@ -949,7 +949,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef args_generator_op() -> list:\n return [\n {\n 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n },\n ]\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-flip-coin-op": { @@ -969,7 +969,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0, 1) == 0 else 'tails'\n return result\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-struct": { @@ -989,7 +989,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_struct(struct: dict):\n print(struct)\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text": { @@ -1009,7 +1009,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-2": { @@ -1029,7 +1029,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-3": { @@ -1049,7 +1049,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-4": { @@ -1069,7 +1069,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-5": { @@ -1089,7 +1089,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-6": { @@ -1109,7 +1109,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-7": { @@ -1129,7 +1129,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-8": { @@ -1149,7 +1149,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-print-text-9": { @@ -1169,7 +1169,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg: {msg}, msg2: {msg2}')\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml index 1d79405a07..f12bc94a80 100644 --- a/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml +++ b/frontend/mock-backend/data/v2/pipeline/pipeline_with_loops_and_conditions.yaml @@ -588,7 +588,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.7 + image: python:3.9 exec-args-generator-op-2: container: args: @@ -617,7 +617,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.7 + image: python:3.9 exec-flip-coin-op: container: args: @@ -645,7 +645,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.7 + image: python:3.9 exec-print-struct: container: args: @@ -671,7 +671,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_struct(struct: dict):\n print(struct)\n\n" - image: python:3.7 + image: python:3.9 exec-print-text: container: args: @@ -698,7 +698,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-2: container: args: @@ -725,7 +725,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-3: container: args: @@ -752,7 +752,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-4: container: args: @@ -779,7 +779,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-5: container: args: @@ -806,7 +806,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-6: container: args: @@ -833,7 +833,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-7: container: args: @@ -860,7 +860,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-8: container: args: @@ -887,7 +887,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-9: container: args: @@ -914,7 +914,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 pipelineInfo: name: pipeline-with-loops-and-conditions-multi-layers root: diff --git a/frontend/mock-backend/data/v2/pipeline/pipeline_with_various_io_types.yaml b/frontend/mock-backend/data/v2/pipeline/pipeline_with_various_io_types.yaml index ddaee4c210..8ede2e2f9b 100644 --- a/frontend/mock-backend/data/v2/pipeline/pipeline_with_various_io_types.yaml +++ b/frontend/mock-backend/data/v2/pipeline/pipeline_with_various_io_types.yaml @@ -87,7 +87,7 @@ components: parameters: output_1: parameterType: NUMBER_INTEGER -defaultPipelineRoot: dummy_root +defaultPipelineRoot: minio://dummy_root deploymentSpec: executors: exec-downstream: diff --git a/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json b/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json index f75a60e1bc..4bbf2bcb0d 100644 --- a/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json +++ b/frontend/mock-backend/data/v2/pipeline/protobuf_value_params_v2.json @@ -48,7 +48,7 @@ "program_path=$(mktemp -d)\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\npython3 -m kfp.components.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef print_params(\n int_param: int = 1234, \n double_param: float = 56.78, \n string_param: str = 'lorem ipsum', \n bool_param: bool = True, \n list_string_param: List[str] = ['lorem', 'ipsum'], \n list_int_param: List[int] = [123, 456, 789], \n struct_param: Dict[str, int] = { 'key_1': 12345, 'key_2': 6789 }):\n print(\"int_param: \", int_param)\n print(\"double_param: \", double_param)\n print(\"string_param: \", string_param)\n print(\"bool_param: \", bool_param)\n print(\"list_string_param: \", list_string_param)\n print(\"list_int_param: \", list_int_param)\n print(\"struct_param: \", struct_param)\n\n" ], - "image": "python:3.7" + "image": "python:3.9" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json index 15e5f4d424..00e235989e 100644 --- a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json +++ b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.json @@ -327,7 +327,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef convert_csv_to_apache_parquet(\n data_path,\n output_data_path,\n):\n '''Converts CSV table to Apache Parquet.\n\n [Apache Parquet](https://parquet.apache.org/)\n\n Annotations:\n author: Alexey Volkov \n '''\n from pyarrow import csv, parquet\n\n table = csv.read_csv(data_path)\n parquet.write_table(table, output_data_path)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Convert csv to apache parquet', description='Converts CSV table to Apache Parquet.\\n\\n [Apache Parquet](https://parquet.apache.org/)\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--output-data\", dest=\"output_data_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n_output_files = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = convert_csv_to_apache_parquet(**_parsed_args)\n\n_output_serializers = [\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except OSError:\n pass\n with open(output_file, 'w') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-xgboost-predict": { @@ -351,7 +351,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path, # Also supports LibSVM\n model_path,\n predictions_path,\n label_column = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in CSV format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column: Column containing the label data.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n df = pandas.read_csv(\n data_path,\n )\n\n if label_column is not None:\n df = df.drop(columns=[df.columns[label_column]])\n\n testing_data = xgboost.DMatrix(\n data=df,\n )\n\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(testing_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in CSV format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column: Column containing the label data.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-xgboost-predict-2": { @@ -375,7 +375,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path,\n model_path,\n predictions_path,\n label_column_name = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in Apache Parquet format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n # Loading data\n df = pandas.read_parquet(data_path)\n if label_column_name:\n df = df.drop(columns=[label_column_name])\n\n evaluation_data = xgboost.DMatrix(\n data=df,\n )\n\n # Training\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(evaluation_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in Apache Parquet format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column-name\", dest=\"label_column_name\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-xgboost-predict-3": { @@ -399,7 +399,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path,\n model_path,\n predictions_path,\n label_column_name = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in Apache Parquet format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n # Loading data\n df = pandas.read_parquet(data_path)\n if label_column_name:\n df = df.drop(columns=[label_column_name])\n\n evaluation_data = xgboost.DMatrix(\n data=df,\n )\n\n # Training\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(evaluation_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in Apache Parquet format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column_name: Optional. Name of the column containing the label data that is excluded during the prediction.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column-name\", dest=\"label_column_name\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-xgboost-predict-4": { @@ -423,7 +423,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_predict(\n data_path, # Also supports LibSVM\n model_path,\n predictions_path,\n label_column = None,\n):\n '''Make predictions using a trained XGBoost model.\n\n Args:\n data_path: Path for the feature data in CSV format.\n model_path: Path for the trained model in binary XGBoost format.\n predictions_path: Output path for the predictions.\n label_column: Column containing the label data.\n\n Annotations:\n author: Alexey Volkov \n '''\n from pathlib import Path\n\n import numpy\n import pandas\n import xgboost\n\n df = pandas.read_csv(\n data_path,\n )\n\n if label_column is not None:\n df = df.drop(columns=[df.columns[label_column]])\n\n testing_data = xgboost.DMatrix(\n data=df,\n )\n\n model = xgboost.Booster(model_file=model_path)\n\n predictions = model.predict(testing_data)\n\n Path(predictions_path).parent.mkdir(parents=True, exist_ok=True)\n numpy.savetxt(predictions_path, predictions)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost predict', description='Make predictions using a trained XGBoost model.\\n\\n Args:\\n data_path: Path for the feature data in CSV format.\\n model_path: Path for the trained model in binary XGBoost format.\\n predictions_path: Output path for the predictions.\\n label_column: Column containing the label data.\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--data\", dest=\"data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_predict(**_parsed_args)\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-xgboost-train": { @@ -459,7 +459,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_train(\n training_data_path, # Also supports LibSVM\n model_path,\n model_config_path,\n starting_model_path = None,\n\n label_column = 0,\n num_iterations = 10,\n booster_params = None,\n\n # Booster parameters\n objective = 'reg:squarederror',\n booster = 'gbtree',\n learning_rate = 0.3,\n min_split_loss = 0,\n max_depth = 6,\n):\n '''Train an XGBoost model.\n\n Args:\n training_data_path: Path for the training data in CSV format.\n model_path: Output path for the trained model in binary XGBoost format.\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\n starting_model_path: Path for the existing trained model to start from.\n label_column: Column containing the label data.\n num_boost_rounds: Number of boosting iterations.\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\n objective: The learning task and the corresponding learning objective.\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\n The most common values are:\n \"reg:squarederror\" - Regression with squared loss (default).\n \"reg:logistic\" - Logistic regression.\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\n\n Annotations:\n author: Alexey Volkov \n '''\n import pandas\n import xgboost\n\n df = pandas.read_csv(\n training_data_path,\n )\n\n training_data = xgboost.DMatrix(\n data=df.drop(columns=[df.columns[label_column]]),\n label=df[df.columns[label_column]],\n )\n\n booster_params = booster_params or {}\n booster_params.setdefault('objective', objective)\n booster_params.setdefault('booster', booster)\n booster_params.setdefault('learning_rate', learning_rate)\n booster_params.setdefault('min_split_loss', min_split_loss)\n booster_params.setdefault('max_depth', max_depth)\n\n starting_model = None\n if starting_model_path:\n starting_model = xgboost.Booster(model_file=starting_model_path)\n\n model = xgboost.train(\n params=booster_params,\n dtrain=training_data,\n num_boost_round=num_iterations,\n xgb_model=starting_model\n )\n\n # Saving the model in binary format\n model.save_model(model_path)\n\n model_config_str = model.save_config()\n with open(model_config_path, 'w') as model_config_file:\n model_config_file.write(model_config_str)\n\nimport json\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost train', description='Train an XGBoost model.\\n\\n Args:\\n training_data_path: Path for the training data in CSV format.\\n model_path: Output path for the trained model in binary XGBoost format.\\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\\n starting_model_path: Path for the existing trained model to start from.\\n label_column: Column containing the label data.\\n num_boost_rounds: Number of boosting iterations.\\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\\n objective: The learning task and the corresponding learning objective.\\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\\n The most common values are:\\n \"reg:squarederror\" - Regression with squared loss (default).\\n \"reg:logistic\" - Logistic regression.\\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--training-data\", dest=\"training_data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--starting-model\", dest=\"starting_model_path\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column\", dest=\"label_column\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--num-iterations\", dest=\"num_iterations\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster-params\", dest=\"booster_params\", type=json.loads, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--objective\", dest=\"objective\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster\", dest=\"booster\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--learning-rate\", dest=\"learning_rate\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--min-split-loss\", dest=\"min_split_loss\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--max-depth\", dest=\"max_depth\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model-config\", dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_train(**_parsed_args)\n" ], - "image": "python:3.7" + "image": "python:3.9" } }, "exec-xgboost-train-2": { @@ -495,7 +495,7 @@ "-c", "def _make_parent_dirs_and_return_path(file_path: str):\n import os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return file_path\n\ndef xgboost_train(\n training_data_path,\n model_path,\n model_config_path,\n label_column_name,\n\n starting_model_path = None,\n\n num_iterations = 10,\n booster_params = None,\n\n # Booster parameters\n objective = 'reg:squarederror',\n booster = 'gbtree',\n learning_rate = 0.3,\n min_split_loss = 0,\n max_depth = 6,\n):\n '''Train an XGBoost model.\n\n Args:\n training_data_path: Path for the training data in Apache Parquet format.\n model_path: Output path for the trained model in binary XGBoost format.\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\n starting_model_path: Path for the existing trained model to start from.\n label_column_name: Name of the column containing the label data.\n num_boost_rounds: Number of boosting iterations.\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\n objective: The learning task and the corresponding learning objective.\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\n The most common values are:\n \"reg:squarederror\" - Regression with squared loss (default).\n \"reg:logistic\" - Logistic regression.\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\n\n Annotations:\n author: Alexey Volkov \n '''\n import pandas\n import xgboost\n\n # Loading data\n df = pandas.read_parquet(training_data_path)\n training_data = xgboost.DMatrix(\n data=df.drop(columns=[label_column_name]),\n label=df[[label_column_name]],\n )\n # Training\n booster_params = booster_params or {}\n booster_params.setdefault('objective', objective)\n booster_params.setdefault('booster', booster)\n booster_params.setdefault('learning_rate', learning_rate)\n booster_params.setdefault('min_split_loss', min_split_loss)\n booster_params.setdefault('max_depth', max_depth)\n\n starting_model = None\n if starting_model_path:\n starting_model = xgboost.Booster(model_file=starting_model_path)\n\n model = xgboost.train(\n params=booster_params,\n dtrain=training_data,\n num_boost_round=num_iterations,\n xgb_model=starting_model\n )\n\n # Saving the model in binary format\n model.save_model(model_path)\n\n model_config_str = model.save_config()\n with open(model_config_path, 'w') as model_config_file:\n model_config_file.write(model_config_str)\n\nimport json\nimport argparse\n_parser = argparse.ArgumentParser(prog='Xgboost train', description='Train an XGBoost model.\\n\\n Args:\\n training_data_path: Path for the training data in Apache Parquet format.\\n model_path: Output path for the trained model in binary XGBoost format.\\n model_config_path: Output path for the internal parameter configuration of Booster as a JSON string.\\n starting_model_path: Path for the existing trained model to start from.\\n label_column_name: Name of the column containing the label data.\\n num_boost_rounds: Number of boosting iterations.\\n booster_params: Parameters for the booster. See https://xgboost.readthedocs.io/en/latest/parameter.html\\n objective: The learning task and the corresponding learning objective.\\n See https://xgboost.readthedocs.io/en/latest/parameter.html#learning-task-parameters\\n The most common values are:\\n \"reg:squarederror\" - Regression with squared loss (default).\\n \"reg:logistic\" - Logistic regression.\\n \"binary:logistic\" - Logistic regression for binary classification, output probability.\\n \"binary:logitraw\" - Logistic regression for binary classification, output score before logistic transformation\\n \"rank:pairwise\" - Use LambdaMART to perform pairwise ranking where the pairwise loss is minimized\\n \"rank:ndcg\" - Use LambdaMART to perform list-wise ranking where Normalized Discounted Cumulative Gain (NDCG) is maximized\\n\\n Annotations:\\n author: Alexey Volkov ')\n_parser.add_argument(\"--training-data\", dest=\"training_data_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--label-column-name\", dest=\"label_column_name\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--starting-model\", dest=\"starting_model_path\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--num-iterations\", dest=\"num_iterations\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster-params\", dest=\"booster_params\", type=json.loads, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--objective\", dest=\"objective\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--booster\", dest=\"booster\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--learning-rate\", dest=\"learning_rate\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--min-split-loss\", dest=\"min_split_loss\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--max-depth\", dest=\"max_depth\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model\", dest=\"model_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--model-config\", dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs = xgboost_train(**_parsed_args)\n" ], - "image": "python:3.7" + "image": "python:3.9" } } } diff --git a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml index 982676b2de..c5612c190d 100644 --- a/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml +++ b/frontend/mock-backend/data/v2/pipeline/xgboost_sample_pipeline.yaml @@ -198,7 +198,7 @@ components: artifactType: schemaTitle: system.Artifact schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root +defaultPipelineRoot: mino://dummy_root deploymentSpec: executors: exec-chicago-taxi-trips-dataset: @@ -253,7 +253,7 @@ deploymentSpec: \ try:\n os.makedirs(os.path.dirname(output_file))\n except\ \ OSError:\n pass\n with open(output_file, 'w') as f:\n \ \ f.write(_output_serializers[idx](_outputs[idx]))\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict: container: args: @@ -304,7 +304,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict-2: container: args: @@ -358,7 +358,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict-3: container: args: @@ -412,7 +412,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict-4: container: args: @@ -463,7 +463,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-train: container: args: @@ -571,7 +571,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-train-2: container: args: @@ -679,7 +679,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 pipelineInfo: name: xgboost-sample-pipeline root: diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d259ae6ec0..0593ec2cb3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -14865,7 +14865,7 @@ }, "camelcase-css": { "version": "2.0.1", - "resolved": "https://artifactory.twitter.biz:443/api/npm/js-virtual/camelcase-css/-/camelcase-css-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", "integrity": "sha1-7pePaUeRTMMMa0R0G27R338EP9U=", "dev": true }, @@ -14887,9 +14887,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001373", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001373.tgz", - "integrity": "sha512-pJYArGHrPp3TUqQzFYRmP/lwJlj8RCbVe3Gd3eJQkAV8SAC6b19XS9BjMvRdvaS8RMkaTN8ZhoHP6S1y8zzwEQ==", + "version": "1.0.30001519", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001519.tgz", + "integrity": "sha512-0QHgqR+Jv4bxHMp8kZ1Kn8CH55OikjKJ6JmKkZYP1F3D7w+lnFXF70nG5eNfsZS89jadi5Ywy5UCSKLAglIRkg==", "dev": true }, "capture-exit": { @@ -21688,7 +21688,7 @@ }, "html-tags": { "version": "3.1.0", - "resolved": "https://artifactory.twitter.biz:443/api/npm/js-virtual/html-tags/-/html-tags-3.1.0.tgz", + "resolved": "https://registry.npmjs.org/html-tags/-/html-tags-3.1.0.tgz", "integrity": "sha1-e15vfmZen7QfMAB+2eDUHpf7IUA=", "dev": true }, @@ -40083,4 +40083,4 @@ "dev": true } } -} +} \ No newline at end of file diff --git a/frontend/server/package-lock.json b/frontend/server/package-lock.json index 8a63202565..a28bda218e 100644 --- a/frontend/server/package-lock.json +++ b/frontend/server/package-lock.json @@ -2,10 +2,27 @@ "requires": true, "lockfileVersion": 1, "dependencies": { + "@aws-crypto/crc32": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", + "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", + "requires": { + "@aws-crypto/util": "^3.0.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^1.11.1" + }, + "dependencies": { + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + } + } + }, "@aws-crypto/ie11-detection": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-2.0.2.tgz", - "integrity": "sha512-5XDMQY98gMAf/WRTic5G++jfmS/VLM0rwpiOpaainKi4L0nqWMSB1SzsrEG5rjFZGYN6ZAefO+/Yta2dFM0kMw==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", + "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", "requires": { "tslib": "^1.11.1" }, @@ -18,15 +35,15 @@ } }, "@aws-crypto/sha256-browser": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-2.0.0.tgz", - "integrity": "sha512-rYXOQ8BFOaqMEHJrLHul/25ckWH6GTJtdLSajhlqGMx0PmSueAuvboCuZCTqEKlxR8CQOwRarxYMZZSYlhRA1A==", - "requires": { - "@aws-crypto/ie11-detection": "^2.0.0", - "@aws-crypto/sha256-js": "^2.0.0", - "@aws-crypto/supports-web-crypto": "^2.0.0", - "@aws-crypto/util": "^2.0.0", - "@aws-sdk/types": "^3.1.0", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", + "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", + "requires": { + "@aws-crypto/ie11-detection": "^3.0.0", + "@aws-crypto/sha256-js": "^3.0.0", + "@aws-crypto/supports-web-crypto": "^3.0.0", + "@aws-crypto/util": "^3.0.0", + "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", "@aws-sdk/util-utf8-browser": "^3.0.0", "tslib": "^1.11.1" @@ -40,12 +57,12 @@ } }, "@aws-crypto/sha256-js": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-2.0.0.tgz", - "integrity": "sha512-VZY+mCY4Nmrs5WGfitmNqXzaE873fcIZDu54cbaDaaamsaTOP1DBImV9F4pICc3EHjQXujyE8jig+PFCaew9ig==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", + "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", "requires": { - "@aws-crypto/util": "^2.0.0", - "@aws-sdk/types": "^3.1.0", + "@aws-crypto/util": "^3.0.0", + "@aws-sdk/types": "^3.222.0", "tslib": "^1.11.1" }, "dependencies": { @@ -57,9 +74,9 @@ } }, "@aws-crypto/supports-web-crypto": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-2.0.2.tgz", - "integrity": "sha512-6mbSsLHwZ99CTOOswvCRP3C+VCWnzBf+1SnbWxzzJ9lR0mA0JnY2JEAhp8rqmTE0GPFy88rrM27ffgp62oErMQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", + "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", "requires": { "tslib": "^1.11.1" }, @@ -72,11 +89,11 @@ } }, "@aws-crypto/util": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-2.0.2.tgz", - "integrity": "sha512-Lgu5v/0e/BcrZ5m/IWqzPUf3UYFTy/PpeED+uc9SWUR1iZQL8XXbGQg10UfllwwBryO3hFF5dizK+78aoXC1eA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", + "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", "requires": { - "@aws-sdk/types": "^3.110.0", + "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-utf8-browser": "^3.0.0", "tslib": "^1.11.1" }, @@ -89,594 +106,608 @@ } }, "@aws-sdk/abort-controller": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/abort-controller/-/abort-controller-3.226.0.tgz", - "integrity": "sha512-cJVzr1xxPBd08voknXvR0RLgtZKGKt6WyDpH/BaPCu3rfSqWCDZKzwqe940eqosjmKrxC6pUZNKASIqHOQ8xxQ==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/abort-controller/-/abort-controller-3.357.0.tgz", + "integrity": "sha512-nQYDJon87quPwt2JZJwUN2GFKJnvE5kWb6tZP4xb5biSGUKBqDQo06oYed7yokatCuCMouIXV462aN0fWODtOw==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/client-cognito-identity": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.241.0.tgz", - "integrity": "sha512-9X/MwcnSwWfB0ijggFjyBWa4gtlUAyI39eBaVSE0AxMcgLlHKedEK6w5F1RrtvWqb7KyJDsyAysVecU4E9zQQQ==", - "requires": { - "@aws-crypto/sha256-browser": "2.0.0", - "@aws-crypto/sha256-js": "2.0.0", - "@aws-sdk/client-sts": "3.241.0", - "@aws-sdk/config-resolver": "3.234.0", - "@aws-sdk/credential-provider-node": "3.241.0", - "@aws-sdk/fetch-http-handler": "3.226.0", - "@aws-sdk/hash-node": "3.226.0", - "@aws-sdk/invalid-dependency": "3.226.0", - "@aws-sdk/middleware-content-length": "3.226.0", - "@aws-sdk/middleware-endpoint": "3.226.0", - "@aws-sdk/middleware-host-header": "3.226.0", - "@aws-sdk/middleware-logger": "3.226.0", - "@aws-sdk/middleware-recursion-detection": "3.226.0", - "@aws-sdk/middleware-retry": "3.235.0", - "@aws-sdk/middleware-serde": "3.226.0", - "@aws-sdk/middleware-signing": "3.226.0", - "@aws-sdk/middleware-stack": "3.226.0", - "@aws-sdk/middleware-user-agent": "3.226.0", - "@aws-sdk/node-config-provider": "3.226.0", - "@aws-sdk/node-http-handler": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/smithy-client": "3.234.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/url-parser": "3.226.0", - "@aws-sdk/util-base64": "3.208.0", - "@aws-sdk/util-body-length-browser": "3.188.0", - "@aws-sdk/util-body-length-node": "3.208.0", - "@aws-sdk/util-defaults-mode-browser": "3.234.0", - "@aws-sdk/util-defaults-mode-node": "3.234.0", - "@aws-sdk/util-endpoints": "3.241.0", - "@aws-sdk/util-retry": "3.229.0", - "@aws-sdk/util-user-agent-browser": "3.226.0", - "@aws-sdk/util-user-agent-node": "3.226.0", - "@aws-sdk/util-utf8-browser": "3.188.0", - "@aws-sdk/util-utf8-node": "3.208.0", - "tslib": "^2.3.1" + "version": "3.359.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.359.0.tgz", + "integrity": "sha512-zb5hSVuyHOXFTjGiqzPhQ/F6Zg4oLffO/NmC3MyvufUzr8yZYmcQzxNU6Jv6WbVmP01OiU4KAozBLMS7URfgzg==", + "requires": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/client-sts": "3.359.0", + "@aws-sdk/config-resolver": "3.357.0", + "@aws-sdk/credential-provider-node": "3.358.0", + "@aws-sdk/fetch-http-handler": "3.357.0", + "@aws-sdk/hash-node": "3.357.0", + "@aws-sdk/invalid-dependency": "3.357.0", + "@aws-sdk/middleware-content-length": "3.357.0", + "@aws-sdk/middleware-endpoint": "3.357.0", + "@aws-sdk/middleware-host-header": "3.357.0", + "@aws-sdk/middleware-logger": "3.357.0", + "@aws-sdk/middleware-recursion-detection": "3.357.0", + "@aws-sdk/middleware-retry": "3.357.0", + "@aws-sdk/middleware-serde": "3.357.0", + "@aws-sdk/middleware-signing": "3.357.0", + "@aws-sdk/middleware-stack": "3.357.0", + "@aws-sdk/middleware-user-agent": "3.357.0", + "@aws-sdk/node-config-provider": "3.357.0", + "@aws-sdk/node-http-handler": "3.357.0", + "@aws-sdk/smithy-client": "3.358.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/url-parser": "3.357.0", + "@aws-sdk/util-base64": "3.310.0", + "@aws-sdk/util-body-length-browser": "3.310.0", + "@aws-sdk/util-body-length-node": "3.310.0", + "@aws-sdk/util-defaults-mode-browser": "3.358.0", + "@aws-sdk/util-defaults-mode-node": "3.358.0", + "@aws-sdk/util-endpoints": "3.357.0", + "@aws-sdk/util-retry": "3.357.0", + "@aws-sdk/util-user-agent-browser": "3.357.0", + "@aws-sdk/util-user-agent-node": "3.357.0", + "@aws-sdk/util-utf8": "3.310.0", + "@smithy/protocol-http": "^1.0.1", + "@smithy/types": "^1.0.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/client-sso": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.241.0.tgz", - "integrity": "sha512-Jm4HR+RYAqKMEYZvvWaq0NYUKKonyInOeubObXH4BLXZpmUBSdYCSjjLdNJY3jkQoxbDVPVMIurVNh5zT5SMRw==", - "requires": { - "@aws-crypto/sha256-browser": "2.0.0", - "@aws-crypto/sha256-js": "2.0.0", - "@aws-sdk/config-resolver": "3.234.0", - "@aws-sdk/fetch-http-handler": "3.226.0", - "@aws-sdk/hash-node": "3.226.0", - "@aws-sdk/invalid-dependency": "3.226.0", - "@aws-sdk/middleware-content-length": "3.226.0", - "@aws-sdk/middleware-endpoint": "3.226.0", - "@aws-sdk/middleware-host-header": "3.226.0", - "@aws-sdk/middleware-logger": "3.226.0", - "@aws-sdk/middleware-recursion-detection": "3.226.0", - "@aws-sdk/middleware-retry": "3.235.0", - "@aws-sdk/middleware-serde": "3.226.0", - "@aws-sdk/middleware-stack": "3.226.0", - "@aws-sdk/middleware-user-agent": "3.226.0", - "@aws-sdk/node-config-provider": "3.226.0", - "@aws-sdk/node-http-handler": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/smithy-client": "3.234.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/url-parser": "3.226.0", - "@aws-sdk/util-base64": "3.208.0", - "@aws-sdk/util-body-length-browser": "3.188.0", - "@aws-sdk/util-body-length-node": "3.208.0", - "@aws-sdk/util-defaults-mode-browser": "3.234.0", - "@aws-sdk/util-defaults-mode-node": "3.234.0", - "@aws-sdk/util-endpoints": "3.241.0", - "@aws-sdk/util-retry": "3.229.0", - "@aws-sdk/util-user-agent-browser": "3.226.0", - "@aws-sdk/util-user-agent-node": "3.226.0", - "@aws-sdk/util-utf8-browser": "3.188.0", - "@aws-sdk/util-utf8-node": "3.208.0", - "tslib": "^2.3.1" + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.358.0.tgz", + "integrity": "sha512-Kc9IsoPIHJfkjDuStyItwQAOpnxw/I9xfF3vvukeN9vkXcRiWeMDhEXACN4L1AYFlU9FHQSRdNwpYTIz7OrD2A==", + "requires": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/config-resolver": "3.357.0", + "@aws-sdk/fetch-http-handler": "3.357.0", + "@aws-sdk/hash-node": "3.357.0", + "@aws-sdk/invalid-dependency": "3.357.0", + "@aws-sdk/middleware-content-length": "3.357.0", + "@aws-sdk/middleware-endpoint": "3.357.0", + "@aws-sdk/middleware-host-header": "3.357.0", + "@aws-sdk/middleware-logger": "3.357.0", + "@aws-sdk/middleware-recursion-detection": "3.357.0", + "@aws-sdk/middleware-retry": "3.357.0", + "@aws-sdk/middleware-serde": "3.357.0", + "@aws-sdk/middleware-stack": "3.357.0", + "@aws-sdk/middleware-user-agent": "3.357.0", + "@aws-sdk/node-config-provider": "3.357.0", + "@aws-sdk/node-http-handler": "3.357.0", + "@aws-sdk/smithy-client": "3.358.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/url-parser": "3.357.0", + "@aws-sdk/util-base64": "3.310.0", + "@aws-sdk/util-body-length-browser": "3.310.0", + "@aws-sdk/util-body-length-node": "3.310.0", + "@aws-sdk/util-defaults-mode-browser": "3.358.0", + "@aws-sdk/util-defaults-mode-node": "3.358.0", + "@aws-sdk/util-endpoints": "3.357.0", + "@aws-sdk/util-retry": "3.357.0", + "@aws-sdk/util-user-agent-browser": "3.357.0", + "@aws-sdk/util-user-agent-node": "3.357.0", + "@aws-sdk/util-utf8": "3.310.0", + "@smithy/protocol-http": "^1.0.1", + "@smithy/types": "^1.0.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/client-sso-oidc": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.241.0.tgz", - "integrity": "sha512-/Ml2QBGpGfUEeBrPzBZhSTBkHuXFD2EAZEIHGCBH4tKaURDI6/FoGI8P1Rl4BzoFt+II/Cr91Eox6YT9EwChsQ==", - "requires": { - "@aws-crypto/sha256-browser": "2.0.0", - "@aws-crypto/sha256-js": "2.0.0", - "@aws-sdk/config-resolver": "3.234.0", - "@aws-sdk/fetch-http-handler": "3.226.0", - "@aws-sdk/hash-node": "3.226.0", - "@aws-sdk/invalid-dependency": "3.226.0", - "@aws-sdk/middleware-content-length": "3.226.0", - "@aws-sdk/middleware-endpoint": "3.226.0", - "@aws-sdk/middleware-host-header": "3.226.0", - "@aws-sdk/middleware-logger": "3.226.0", - "@aws-sdk/middleware-recursion-detection": "3.226.0", - "@aws-sdk/middleware-retry": "3.235.0", - "@aws-sdk/middleware-serde": "3.226.0", - "@aws-sdk/middleware-stack": "3.226.0", - "@aws-sdk/middleware-user-agent": "3.226.0", - "@aws-sdk/node-config-provider": "3.226.0", - "@aws-sdk/node-http-handler": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/smithy-client": "3.234.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/url-parser": "3.226.0", - "@aws-sdk/util-base64": "3.208.0", - "@aws-sdk/util-body-length-browser": "3.188.0", - "@aws-sdk/util-body-length-node": "3.208.0", - "@aws-sdk/util-defaults-mode-browser": "3.234.0", - "@aws-sdk/util-defaults-mode-node": "3.234.0", - "@aws-sdk/util-endpoints": "3.241.0", - "@aws-sdk/util-retry": "3.229.0", - "@aws-sdk/util-user-agent-browser": "3.226.0", - "@aws-sdk/util-user-agent-node": "3.226.0", - "@aws-sdk/util-utf8-browser": "3.188.0", - "@aws-sdk/util-utf8-node": "3.208.0", - "tslib": "^2.3.1" + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.358.0.tgz", + "integrity": "sha512-Gy09fSlhJdGbr8rNNR8EdLaUynB1B34nw8kN1aFT4CdAnjFKxTainqG6Aq4vx64TbMDMhvMYWpNAluvq7UHVhw==", + "requires": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/config-resolver": "3.357.0", + "@aws-sdk/fetch-http-handler": "3.357.0", + "@aws-sdk/hash-node": "3.357.0", + "@aws-sdk/invalid-dependency": "3.357.0", + "@aws-sdk/middleware-content-length": "3.357.0", + "@aws-sdk/middleware-endpoint": "3.357.0", + "@aws-sdk/middleware-host-header": "3.357.0", + "@aws-sdk/middleware-logger": "3.357.0", + "@aws-sdk/middleware-recursion-detection": "3.357.0", + "@aws-sdk/middleware-retry": "3.357.0", + "@aws-sdk/middleware-serde": "3.357.0", + "@aws-sdk/middleware-stack": "3.357.0", + "@aws-sdk/middleware-user-agent": "3.357.0", + "@aws-sdk/node-config-provider": "3.357.0", + "@aws-sdk/node-http-handler": "3.357.0", + "@aws-sdk/smithy-client": "3.358.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/url-parser": "3.357.0", + "@aws-sdk/util-base64": "3.310.0", + "@aws-sdk/util-body-length-browser": "3.310.0", + "@aws-sdk/util-body-length-node": "3.310.0", + "@aws-sdk/util-defaults-mode-browser": "3.358.0", + "@aws-sdk/util-defaults-mode-node": "3.358.0", + "@aws-sdk/util-endpoints": "3.357.0", + "@aws-sdk/util-retry": "3.357.0", + "@aws-sdk/util-user-agent-browser": "3.357.0", + "@aws-sdk/util-user-agent-node": "3.357.0", + "@aws-sdk/util-utf8": "3.310.0", + "@smithy/protocol-http": "^1.0.1", + "@smithy/types": "^1.0.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/client-sts": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.241.0.tgz", - "integrity": "sha512-vmlG8cJzRf8skCtTJbA2wBvD2c3NQ5gZryzJvTKDS06KzBzcEpnjlLseuTekcnOiRNekbFUX5hRu5Zj3N2ReLg==", - "requires": { - "@aws-crypto/sha256-browser": "2.0.0", - "@aws-crypto/sha256-js": "2.0.0", - "@aws-sdk/config-resolver": "3.234.0", - "@aws-sdk/credential-provider-node": "3.241.0", - "@aws-sdk/fetch-http-handler": "3.226.0", - "@aws-sdk/hash-node": "3.226.0", - "@aws-sdk/invalid-dependency": "3.226.0", - "@aws-sdk/middleware-content-length": "3.226.0", - "@aws-sdk/middleware-endpoint": "3.226.0", - "@aws-sdk/middleware-host-header": "3.226.0", - "@aws-sdk/middleware-logger": "3.226.0", - "@aws-sdk/middleware-recursion-detection": "3.226.0", - "@aws-sdk/middleware-retry": "3.235.0", - "@aws-sdk/middleware-sdk-sts": "3.226.0", - "@aws-sdk/middleware-serde": "3.226.0", - "@aws-sdk/middleware-signing": "3.226.0", - "@aws-sdk/middleware-stack": "3.226.0", - "@aws-sdk/middleware-user-agent": "3.226.0", - "@aws-sdk/node-config-provider": "3.226.0", - "@aws-sdk/node-http-handler": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/smithy-client": "3.234.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/url-parser": "3.226.0", - "@aws-sdk/util-base64": "3.208.0", - "@aws-sdk/util-body-length-browser": "3.188.0", - "@aws-sdk/util-body-length-node": "3.208.0", - "@aws-sdk/util-defaults-mode-browser": "3.234.0", - "@aws-sdk/util-defaults-mode-node": "3.234.0", - "@aws-sdk/util-endpoints": "3.241.0", - "@aws-sdk/util-retry": "3.229.0", - "@aws-sdk/util-user-agent-browser": "3.226.0", - "@aws-sdk/util-user-agent-node": "3.226.0", - "@aws-sdk/util-utf8-browser": "3.188.0", - "@aws-sdk/util-utf8-node": "3.208.0", - "fast-xml-parser": "4.0.11", - "tslib": "^2.3.1" + "version": "3.359.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.359.0.tgz", + "integrity": "sha512-zpyui8hXvEUvq8MwzZsm51ni0intvPjtV8dgx10nVJnm605nqrLlAMGqQ1S/UxO7CVmhqWbh5dnGHEc//UJlsw==", + "requires": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/config-resolver": "3.357.0", + "@aws-sdk/credential-provider-node": "3.358.0", + "@aws-sdk/fetch-http-handler": "3.357.0", + "@aws-sdk/hash-node": "3.357.0", + "@aws-sdk/invalid-dependency": "3.357.0", + "@aws-sdk/middleware-content-length": "3.357.0", + "@aws-sdk/middleware-endpoint": "3.357.0", + "@aws-sdk/middleware-host-header": "3.357.0", + "@aws-sdk/middleware-logger": "3.357.0", + "@aws-sdk/middleware-recursion-detection": "3.357.0", + "@aws-sdk/middleware-retry": "3.357.0", + "@aws-sdk/middleware-sdk-sts": "3.357.0", + "@aws-sdk/middleware-serde": "3.357.0", + "@aws-sdk/middleware-signing": "3.357.0", + "@aws-sdk/middleware-stack": "3.357.0", + "@aws-sdk/middleware-user-agent": "3.357.0", + "@aws-sdk/node-config-provider": "3.357.0", + "@aws-sdk/node-http-handler": "3.357.0", + "@aws-sdk/smithy-client": "3.358.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/url-parser": "3.357.0", + "@aws-sdk/util-base64": "3.310.0", + "@aws-sdk/util-body-length-browser": "3.310.0", + "@aws-sdk/util-body-length-node": "3.310.0", + "@aws-sdk/util-defaults-mode-browser": "3.358.0", + "@aws-sdk/util-defaults-mode-node": "3.358.0", + "@aws-sdk/util-endpoints": "3.357.0", + "@aws-sdk/util-retry": "3.357.0", + "@aws-sdk/util-user-agent-browser": "3.357.0", + "@aws-sdk/util-user-agent-node": "3.357.0", + "@aws-sdk/util-utf8": "3.310.0", + "@smithy/protocol-http": "^1.0.1", + "@smithy/types": "^1.0.0", + "fast-xml-parser": "4.2.5", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/config-resolver": { - "version": "3.234.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/config-resolver/-/config-resolver-3.234.0.tgz", - "integrity": "sha512-uZxy4wzllfvgCQxVc+Iqhde0NGAnfmV2hWR6ejadJaAFTuYNvQiRg9IqJy3pkyDPqXySiJ8Bom5PoJfgn55J/A==", - "requires": { - "@aws-sdk/signature-v4": "3.226.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/util-config-provider": "3.208.0", - "@aws-sdk/util-middleware": "3.226.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/config-resolver/-/config-resolver-3.357.0.tgz", + "integrity": "sha512-cukfg0nX7Tzx/xFyH5F4Eyb8DA1ITCGtSQv4vnEjgUop+bkzckuGLKEeBcBhyZY+aw+2C9CVwIHwIMhRm0ul5w==", + "requires": { + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-config-provider": "3.310.0", + "@aws-sdk/util-middleware": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-cognito-identity": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.241.0.tgz", - "integrity": "sha512-e2hlXWG9DH93uVe2wHIUrUOrgZTLzCV3gBd10D3/usSzS4FvVVU7OmidnRPYCLLnt3EvnL5b4REOedO1q8hv8g==", + "version": "3.359.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.359.0.tgz", + "integrity": "sha512-dSuHTucXcjIFsjdOq0HeSk0niWJ7V2hWnwyYh7MCwv43dP9u4V+11boLC6zIrw2Epx++JnIqhggKJAi6l/occw==", "requires": { - "@aws-sdk/client-cognito-identity": "3.241.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/client-cognito-identity": "3.359.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-env": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.226.0.tgz", - "integrity": "sha512-sd8uK1ojbXxaZXlthzw/VXZwCPUtU3PjObOfr3Evj7MPIM2IH8h29foOlggx939MdLQGboJf9gKvLlvKDWtJRA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.357.0.tgz", + "integrity": "sha512-UOecwfqvXgJVqhfWSZ2S44v2Nq2oceW0PQVQp0JAa9opc2rxSVIfyOhPr0yMoPmpyNcP22rgeg6ce70KULYwiA==", "requires": { - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-imds": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-imds/-/credential-provider-imds-3.226.0.tgz", - "integrity": "sha512-//z/COQm2AjYFI1Lb0wKHTQSrvLFTyuKLFQGPJsKS7DPoxGOCKB7hmYerlbl01IDoCxTdyL//TyyPxbZEOQD5Q==", - "requires": { - "@aws-sdk/node-config-provider": "3.226.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/url-parser": "3.226.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-imds/-/credential-provider-imds-3.357.0.tgz", + "integrity": "sha512-upw/bfsl7/WydT6gM0lBuR4Ipp4fzYm/E3ObFr0Mg5OkgVPt5ZJE+eeFTvwCpDdBSTKs4JfrK6/iEK8A23Q1jQ==", + "requires": { + "@aws-sdk/node-config-provider": "3.357.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/url-parser": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-ini": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.241.0.tgz", - "integrity": "sha512-CI+mu6h74Kzmscw35TvNkc/wYHsHPGAwP7humSHoWw53H9mVw21Ggft/dT1iFQQZWQ8BNXkzuXlNo1IlqwMgOA==", - "requires": { - "@aws-sdk/credential-provider-env": "3.226.0", - "@aws-sdk/credential-provider-imds": "3.226.0", - "@aws-sdk/credential-provider-process": "3.226.0", - "@aws-sdk/credential-provider-sso": "3.241.0", - "@aws-sdk/credential-provider-web-identity": "3.226.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/shared-ini-file-loader": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.358.0.tgz", + "integrity": "sha512-Blmw4bhGxpaYvPmrbRKAltqnNDDSf6ZegNqJasc5OWvAlHJNvB/hYPmyQN0oFy79BXn7PbBip1QaLWaEhJvpAA==", + "requires": { + "@aws-sdk/credential-provider-env": "3.357.0", + "@aws-sdk/credential-provider-imds": "3.357.0", + "@aws-sdk/credential-provider-process": "3.357.0", + "@aws-sdk/credential-provider-sso": "3.358.0", + "@aws-sdk/credential-provider-web-identity": "3.357.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/shared-ini-file-loader": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-node": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.241.0.tgz", - "integrity": "sha512-08zPQcD5o9brQmzEipWHeHgU85aQcEF8MWLfpeyjO6e1/l7ysQ35NsS+PYtv77nLpGCx/X+ZuW/KXWoRrbw77w==", - "requires": { - "@aws-sdk/credential-provider-env": "3.226.0", - "@aws-sdk/credential-provider-imds": "3.226.0", - "@aws-sdk/credential-provider-ini": "3.241.0", - "@aws-sdk/credential-provider-process": "3.226.0", - "@aws-sdk/credential-provider-sso": "3.241.0", - "@aws-sdk/credential-provider-web-identity": "3.226.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/shared-ini-file-loader": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.358.0.tgz", + "integrity": "sha512-iLjyRNOT0ycdLqkzXNW+V2zibVljkLjL8j45FpK6mNrAwc/Ynr7EYuRRp5OuRiiYDO3ZoneAxpBJQ5SqmK2Jfg==", + "requires": { + "@aws-sdk/credential-provider-env": "3.357.0", + "@aws-sdk/credential-provider-imds": "3.357.0", + "@aws-sdk/credential-provider-ini": "3.358.0", + "@aws-sdk/credential-provider-process": "3.357.0", + "@aws-sdk/credential-provider-sso": "3.358.0", + "@aws-sdk/credential-provider-web-identity": "3.357.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/shared-ini-file-loader": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-process": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.226.0.tgz", - "integrity": "sha512-iUDMdnrTvbvaCFhWwqyXrhvQ9+ojPqPqXhwZtY1X/Qaz+73S9gXBPJHZaZb2Ke0yKE1Ql3bJbKvmmxC/qLQMng==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.357.0.tgz", + "integrity": "sha512-qFWWilFPsc2hR7O0KIhwcE78w+pVIK+uQR6MQMfdRyxUndgiuCorJwVjedc3yZtmnoELHF34j+m8whTBXv9E7Q==", "requires": { - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/shared-ini-file-loader": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/shared-ini-file-loader": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-sso": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.241.0.tgz", - "integrity": "sha512-6Bjd6eEIrVomRTrPrM4dlxusQm+KMJ9hLYKECCpFkwDKIK+pTgZNLRtQdalHyzwneHJPdimrm8cOv1kUQ8hPoA==", - "requires": { - "@aws-sdk/client-sso": "3.241.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/shared-ini-file-loader": "3.226.0", - "@aws-sdk/token-providers": "3.241.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.358.0.tgz", + "integrity": "sha512-hKu5NshKohSDoHaXKyeCW88J8dBt4TMljrL+WswTMifuThO9ptyMq4PCdl4z7CNjIq6zo3ftc/uNf8TY7Ga8+w==", + "requires": { + "@aws-sdk/client-sso": "3.358.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/shared-ini-file-loader": "3.357.0", + "@aws-sdk/token-providers": "3.358.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-provider-web-identity": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.226.0.tgz", - "integrity": "sha512-CCpv847rLB0SFOHz2igvUMFAzeT2fD3YnY4C8jltuJoEkn0ITn1Hlgt13nTJ5BUuvyti2mvyXZHmNzhMIMrIlw==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.357.0.tgz", + "integrity": "sha512-0KRRAFrXy5HJe2vqnCWCoCS+fQw7IoIj3KQsuURJMW4F+ifisxCgEsh3brJ2LQlN4ElWTRJhlrDHNZ/pd61D4w==", "requires": { - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/credential-providers": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.241.0.tgz", - "integrity": "sha512-J3Q45t1o35OhUI6gWks7rmosPT+mFWXiaHl2LST509Ovjwx6SFs2PvbGP6n7xqUzxyq5Rk6FzZBwB8ItuAa6Qw==", - "requires": { - "@aws-sdk/client-cognito-identity": "3.241.0", - "@aws-sdk/client-sso": "3.241.0", - "@aws-sdk/client-sts": "3.241.0", - "@aws-sdk/credential-provider-cognito-identity": "3.241.0", - "@aws-sdk/credential-provider-env": "3.226.0", - "@aws-sdk/credential-provider-imds": "3.226.0", - "@aws-sdk/credential-provider-ini": "3.241.0", - "@aws-sdk/credential-provider-node": "3.241.0", - "@aws-sdk/credential-provider-process": "3.226.0", - "@aws-sdk/credential-provider-sso": "3.241.0", - "@aws-sdk/credential-provider-web-identity": "3.226.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/shared-ini-file-loader": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.359.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.359.0.tgz", + "integrity": "sha512-fwfdqoJihRUbk3KEYv8IfWRFI+cNQfXfVHLtDEcW3tCU8lqsL920YSEjqMuWGrWLp8dWESDX5C3wZugur0lnTQ==", + "requires": { + "@aws-sdk/client-cognito-identity": "3.359.0", + "@aws-sdk/client-sso": "3.358.0", + "@aws-sdk/client-sts": "3.359.0", + "@aws-sdk/credential-provider-cognito-identity": "3.359.0", + "@aws-sdk/credential-provider-env": "3.357.0", + "@aws-sdk/credential-provider-imds": "3.357.0", + "@aws-sdk/credential-provider-ini": "3.358.0", + "@aws-sdk/credential-provider-node": "3.358.0", + "@aws-sdk/credential-provider-process": "3.357.0", + "@aws-sdk/credential-provider-sso": "3.358.0", + "@aws-sdk/credential-provider-web-identity": "3.357.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" + }, + "dependencies": { + "tslib": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" + } + } + }, + "@aws-sdk/eventstream-codec": { + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-codec/-/eventstream-codec-3.357.0.tgz", + "integrity": "sha512-bqenTHG6GH6aCk/Il+ooWXVVAZuc8lOgVEy9bE2hI49oVqT8zSuXxQB+w1WWyZoAOPcelsjayB1wfPub8VDBxQ==", + "requires": { + "@aws-crypto/crc32": "3.0.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-hex-encoding": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/fetch-http-handler": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/fetch-http-handler/-/fetch-http-handler-3.226.0.tgz", - "integrity": "sha512-JewZPMNEBXfi1xVnRa7pVtK/zgZD8/lQ/YnD8pq79WuMa2cwyhDtr8oqCoqsPW+WJT5ScXoMtuHxN78l8eKWgg==", - "requires": { - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/querystring-builder": "3.226.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/util-base64": "3.208.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/fetch-http-handler/-/fetch-http-handler-3.357.0.tgz", + "integrity": "sha512-5sPloTO8y8fAnS/6/Sfp/aVoL9zuhzkLdWBORNzMazdynVNEzWKWCPZ27RQpgkaCDHiXjqUY4kfuFXAGkvFfDQ==", + "requires": { + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/querystring-builder": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-base64": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/hash-node": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/hash-node/-/hash-node-3.226.0.tgz", - "integrity": "sha512-MdlJhJ9/Espwd0+gUXdZRsHuostB2WxEVAszWxobP0FTT9PnicqnfK7ExmW+DUAc0ywxtEbR3e0UND65rlSTVw==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/hash-node/-/hash-node-3.357.0.tgz", + "integrity": "sha512-fq3LS9AxHKb7dTZkm6iM1TrGk6XOTZz96iEZPME1+vjiSEXGWuebHt87q92n+KozVGRypn9MId3lHOPBBjygNQ==", "requires": { - "@aws-sdk/types": "3.226.0", - "@aws-sdk/util-buffer-from": "3.208.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-buffer-from": "3.310.0", + "@aws-sdk/util-utf8": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/invalid-dependency": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/invalid-dependency/-/invalid-dependency-3.226.0.tgz", - "integrity": "sha512-QXOYFmap8g9QzRjumcRCIo2GEZkdCwd7ePQW0OABWPhKHzlJ74vvBxywjU3s39EEBEluWXtZ7Iufg6GxZM4ifw==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/invalid-dependency/-/invalid-dependency-3.357.0.tgz", + "integrity": "sha512-HnCYZczf0VdyxMVMMxmA3QJAyyPSFbcMtZzgKbxVTWTG7GKpQe0psWZu/7O2Nk31mKg6vEUdiP1FylqLBsgMOA==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/is-array-buffer": { - "version": "3.201.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/is-array-buffer/-/is-array-buffer-3.201.0.tgz", - "integrity": "sha512-UPez5qLh3dNgt0DYnPD/q0mVJY84rA17QE26hVNOW3fAji8W2wrwrxdacWOxyXvlxWsVRcKmr+lay1MDqpAMfg==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/is-array-buffer/-/is-array-buffer-3.310.0.tgz", + "integrity": "sha512-urnbcCR+h9NWUnmOtet/s4ghvzsidFmspfhYaHAmSRdy9yDjdjBJMFjjsn85A1ODUktztm+cVncXjQ38WCMjMQ==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-content-length": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-content-length/-/middleware-content-length-3.226.0.tgz", - "integrity": "sha512-ksUzlHJN2JMuyavjA46a4sctvnrnITqt2tbGGWWrAuXY1mel2j+VbgnmJUiwHKUO6bTFBBeft5Vd1TSOb4JmiA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-content-length/-/middleware-content-length-3.357.0.tgz", + "integrity": "sha512-zQOFEyzOXAgN4M54tYNWGxKxnyzY0WwYDTFzh9riJRmxN1hTEKHUKmze4nILIf5rkQmOG4kTf1qmfazjkvZAhw==", "requires": { - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-endpoint": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint/-/middleware-endpoint-3.226.0.tgz", - "integrity": "sha512-EvLFafjtUxTT0AC9p3aBQu1/fjhWdIeK58jIXaNFONfZ3F8QbEYUPuF/SqZvJM6cWfOO9qwYKkRDbCSTYhprIg==", - "requires": { - "@aws-sdk/middleware-serde": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/signature-v4": "3.226.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/url-parser": "3.226.0", - "@aws-sdk/util-config-provider": "3.208.0", - "@aws-sdk/util-middleware": "3.226.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint/-/middleware-endpoint-3.357.0.tgz", + "integrity": "sha512-ScJi0SL8X/Lyi0Fp5blg0QN/Z6PoRwV/ZJXd8dQkXSznkbSvJHfqPP0xk/w3GcQ1TKsu5YEPfeYy8ejcq+7Pgg==", + "requires": { + "@aws-sdk/middleware-serde": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/url-parser": "3.357.0", + "@aws-sdk/util-middleware": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-host-header": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.226.0.tgz", - "integrity": "sha512-haVkWVh6BUPwKgWwkL6sDvTkcZWvJjv8AgC8jiQuSl8GLZdzHTB8Qhi3IsfFta9HAuoLjxheWBE5Z/L0UrfhLA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.357.0.tgz", + "integrity": "sha512-HuGLcP7JP1qJ5wGT9GSlEknDaTSnOzHY4T6IPFuvFjAy3PvY5siQNm6+VRqdVS+n6/kzpL3JP5sAVM3aoxHT6Q==", "requires": { - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-logger": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.226.0.tgz", - "integrity": "sha512-m9gtLrrYnpN6yckcQ09rV7ExWOLMuq8mMPF/K3DbL/YL0TuILu9i2T1W+JuxSX+K9FMG2HrLAKivE/kMLr55xA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.357.0.tgz", + "integrity": "sha512-dncT3tr+lZ9+duZo52rASgO6AKVwRcsc2/T93gmaYVrJqI6WWAwQ7yML5s72l9ZjQ5LZ+4jjrgtlufavAS0eCg==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-recursion-detection": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.226.0.tgz", - "integrity": "sha512-mwRbdKEUeuNH5TEkyZ5FWxp6bL2UC1WbY+LDv6YjHxmSMKpAoOueEdtU34PqDOLrpXXxIGHDFmjeGeMfktyEcA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.357.0.tgz", + "integrity": "sha512-AXC54IeDS3jC1dbbkYHML4STvBPcKZ4IJTWdjEK1RCOgqXd0Ze1cE1e21wyj1tM6prF03zLyvpBd+3TS++nqfA==", "requires": { - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-retry": { - "version": "3.235.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-retry/-/middleware-retry-3.235.0.tgz", - "integrity": "sha512-50WHbJGpD3SNp9763MAlHqIhXil++JdQbKejNpHg7HsJne/ao3ub+fDOfx//mMBjpzBV25BGd5UlfL6blrClSg==", - "requires": { - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/service-error-classification": "3.229.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/util-middleware": "3.226.0", - "@aws-sdk/util-retry": "3.229.0", - "tslib": "^2.3.1", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-retry/-/middleware-retry-3.357.0.tgz", + "integrity": "sha512-ZCbXCYv3nglQqwREYxxpclrnR9MYPAnHlLcC8e9PbApqxGnaZdhoywxoqbgqT3hf/RM7kput4vEHDl1fyymcRQ==", + "requires": { + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/service-error-classification": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-middleware": "3.357.0", + "@aws-sdk/util-retry": "3.357.0", + "tslib": "^2.5.0", "uuid": "^8.3.2" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" }, "uuid": { "version": "8.3.2", @@ -686,573 +717,597 @@ } }, "@aws-sdk/middleware-sdk-sts": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.226.0.tgz", - "integrity": "sha512-NN9T/qoSD1kZvAT+VLny3NnlqgylYQcsgV3rvi/8lYzw/G/2s8VS6sm/VTWGGZhx08wZRv20MWzYu3bftcyqUg==", - "requires": { - "@aws-sdk/middleware-signing": "3.226.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/signature-v4": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.357.0.tgz", + "integrity": "sha512-Ng2VjLrPiL02QOcs1qs9jG2boO4Gn+v3VIbOJLG4zXcfbSq55iIWtlmr2ljfw9vP5aLhWtcODfmKHS5Bp+019Q==", + "requires": { + "@aws-sdk/middleware-signing": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-serde": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-serde/-/middleware-serde-3.226.0.tgz", - "integrity": "sha512-nPuOOAkSfx9TxzdKFx0X2bDlinOxGrqD7iof926K/AEflxGD1DBdcaDdjlYlPDW2CVE8LV/rAgbYuLxh/E/1VA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-serde/-/middleware-serde-3.357.0.tgz", + "integrity": "sha512-bGI4kYuuEsFjlANbyJLyy4AovETnyf/SukgLOG7Qjbua+ZGuzvRhMsk21mBKKGrnsTO4PmtieJo6xClThGAN8g==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-signing": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.226.0.tgz", - "integrity": "sha512-E6HmtPcl+IjYDDzi1xI2HpCbBq2avNWcjvCriMZWuTAtRVpnA6XDDGW5GY85IfS3A8G8vuWqEVPr8JcYUcjfew==", - "requires": { - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/signature-v4": "3.226.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/util-middleware": "3.226.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.357.0.tgz", + "integrity": "sha512-yB9ewEqI6Fw1OrmKFrUypbCqN5ijk06UGPochybamMuPxxkwMT3bnrm7eezsCA+TZbJyKhpffpyobwuv+xGNrA==", + "requires": { + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/signature-v4": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-middleware": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-stack": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-stack/-/middleware-stack-3.226.0.tgz", - "integrity": "sha512-85wF29LvPvpoed60fZGDYLwv1Zpd/cM0C22WSSFPw1SSJeqO4gtFYyCg2squfT3KI6kF43IIkOCJ+L7GtryPug==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-stack/-/middleware-stack-3.357.0.tgz", + "integrity": "sha512-nNV+jfwGwmbOGZujAY/U8AW3EbVlxa9DJDLz3TPp/39o6Vu5KEzHJyDDNreo2k9V/TMvV+nOzHafufgPdagv7w==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/middleware-user-agent": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.226.0.tgz", - "integrity": "sha512-N1WnfzCW1Y5yWhVAphf8OPGTe8Df3vmV7/LdsoQfmpkCZgLZeK2o0xITkUQhRj1mbw7yp8tVFLFV3R2lMurdAQ==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.357.0.tgz", + "integrity": "sha512-M/CsAXjGblZS4rEbMb0Dn9IXbfq4EjVaTHBfvuILU/dKRppWvjnm2lRtqCZ+LIT3ATbAjA3/dY7dWsjxQWwijA==", "requires": { - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-endpoints": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/node-config-provider": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/node-config-provider/-/node-config-provider-3.226.0.tgz", - "integrity": "sha512-B8lQDqiRk7X5izFEUMXmi8CZLOKCTWQJU9HQf3ako+sF0gexo4nHN3jhoRWyLtcgC5S3on/2jxpAcqtm7kuY3w==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/node-config-provider/-/node-config-provider-3.357.0.tgz", + "integrity": "sha512-kwBIzKCaW3UWqLdELhy7TcN8itNMOjbzga530nalFILMvn2IxrkdKQhNgxGBXy6QK6kCOtH6OmcrG3/oZkLwig==", "requires": { - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/shared-ini-file-loader": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/shared-ini-file-loader": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/node-http-handler": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/node-http-handler/-/node-http-handler-3.226.0.tgz", - "integrity": "sha512-xQCddnZNMiPmjr3W7HYM+f5ir4VfxgJh37eqZwX6EZmyItFpNNeVzKUgA920ka1VPz/ZUYB+2OFGiX3LCLkkaA==", - "requires": { - "@aws-sdk/abort-controller": "3.226.0", - "@aws-sdk/protocol-http": "3.226.0", - "@aws-sdk/querystring-builder": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/node-http-handler/-/node-http-handler-3.357.0.tgz", + "integrity": "sha512-uoab4xIJux+Q9hQ9A/vWEAjojtBQ0U4K7xEQVa0BXEv7MHH5zv51H+VtrelU1Ed6hsHq4Sx0bxBMFpbbWhNyjA==", + "requires": { + "@aws-sdk/abort-controller": "3.357.0", + "@aws-sdk/protocol-http": "3.357.0", + "@aws-sdk/querystring-builder": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/property-provider": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/property-provider/-/property-provider-3.226.0.tgz", - "integrity": "sha512-TsljjG+Sg0LmdgfiAlWohluWKnxB/k8xenjeozZfzOr5bHmNHtdbWv6BtNvD/R83hw7SFXxbJHlD5H4u9p2NFg==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/property-provider/-/property-provider-3.357.0.tgz", + "integrity": "sha512-im4W0u8WaYxG7J7ko4Xl3OEzK3Mrm1Rz6/txTGe6hTIHlyUISu1ekOQJXK6XYPqNMn8v1G3BiQREoRXUEJFbHg==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/protocol-http": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/protocol-http/-/protocol-http-3.226.0.tgz", - "integrity": "sha512-zWkVqiTA9RXL6y0hhfZc9bcU4DX2NI6Hw9IhQmSPeM59mdbPjJlY4bLlMr5YxywqO3yQ/ylNoAfrEzrDjlOSRg==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/protocol-http/-/protocol-http-3.357.0.tgz", + "integrity": "sha512-w1JHiI50VEea7duDeAspUiKJmmdIQblvRyjVMOqWA6FIQAyDVuEiPX7/MdQr0ScxhtRQxHbP0I4MFyl7ctRQvA==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/querystring-builder": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/querystring-builder/-/querystring-builder-3.226.0.tgz", - "integrity": "sha512-LVurypuNeotO4lmirKXRC4NYrZRAyMJXuwO0f2a5ZAUJCjauwYrifKue6yCfU7bls7gut7nfcR6B99WBYpHs3g==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/querystring-builder/-/querystring-builder-3.357.0.tgz", + "integrity": "sha512-aQcicqB6Y2cNaXPPwunz612a01SMiQQPsdz632F/3Lzn0ua82BJKobHOtaiTUlmVJ5Q4/EAeNfwZgL7tTUNtDQ==", "requires": { - "@aws-sdk/types": "3.226.0", - "@aws-sdk/util-uri-escape": "3.201.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-uri-escape": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/querystring-parser": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/querystring-parser/-/querystring-parser-3.226.0.tgz", - "integrity": "sha512-FzB+VrQ47KAFxiPt2YXrKZ8AOLZQqGTLCKHzx4bjxGmwgsjV8yIbtJiJhZLMcUQV4LtGeIY9ixIqQhGvnZHE4A==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/querystring-parser/-/querystring-parser-3.357.0.tgz", + "integrity": "sha512-Svvq+atRNP9s2VxiklcUNgCzmt3T5kfs7X2C+yjmxHvOQTPjLNaNGbfC/vhjOK7aoXw0h+lBac48r5ymx1PbQA==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/service-error-classification": { - "version": "3.229.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/service-error-classification/-/service-error-classification-3.229.0.tgz", - "integrity": "sha512-dnzWWQ0/NoWMUZ5C0DW3dPm0wC1O76Y/SpKbuJzWPkx1EYy6r8p32Ly4D9vUzrKDbRGf48YHIF2kOkBmu21CLg==" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/service-error-classification/-/service-error-classification-3.357.0.tgz", + "integrity": "sha512-VuXeL4g5vKO9HjgCZlxmH8Uv1FcqUSjmbPpQkbNtYIDck6u0qzM0rG+n0/1EjyQbPSr3MhW/pkWs5nx2Nljlyg==" }, "@aws-sdk/shared-ini-file-loader": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/shared-ini-file-loader/-/shared-ini-file-loader-3.226.0.tgz", - "integrity": "sha512-661VQefsARxVyyV2FX9V61V+nNgImk7aN2hYlFKla6BCwZfMng+dEtD0xVGyg1PfRw0qvEv5LQyxMVgHcUSevA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/shared-ini-file-loader/-/shared-ini-file-loader-3.357.0.tgz", + "integrity": "sha512-ceyqM4XxQe0Plb/oQAD2t1UOV2Iy4PFe1oAGM8dfJzYrRKu7zvMwru7/WaB3NYq+/mIY6RU+jjhRmjQ3GySVqA==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/signature-v4": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4/-/signature-v4-3.226.0.tgz", - "integrity": "sha512-/R5q5agdPd7HJB68XMzpxrNPk158EHUvkFkuRu5Qf3kkkHebEzWEBlWoVpUe6ss4rP9Tqcue6xPuaftEmhjpYw==", - "requires": { - "@aws-sdk/is-array-buffer": "3.201.0", - "@aws-sdk/types": "3.226.0", - "@aws-sdk/util-hex-encoding": "3.201.0", - "@aws-sdk/util-middleware": "3.226.0", - "@aws-sdk/util-uri-escape": "3.201.0", - "tslib": "^2.3.1" + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4/-/signature-v4-3.357.0.tgz", + "integrity": "sha512-itt4/Jh9FqnzK30qIjXFBvM4J7zN4S/AAqsRMnaX7U4f/MV+1YxQHmzimpdMnsCXXs2jqFqKVRu6DewxJ3nbxg==", + "requires": { + "@aws-sdk/eventstream-codec": "3.357.0", + "@aws-sdk/is-array-buffer": "3.310.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-hex-encoding": "3.310.0", + "@aws-sdk/util-middleware": "3.357.0", + "@aws-sdk/util-uri-escape": "3.310.0", + "@aws-sdk/util-utf8": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/smithy-client": { - "version": "3.234.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/smithy-client/-/smithy-client-3.234.0.tgz", - "integrity": "sha512-8AtR/k4vsFvjXeQbIzq/Wy7Nbk48Ou0wUEeVYPHWHPSU8QamFWORkOwmKtKMfHAyZvmqiAPeQqHFkq+UJhWyyQ==", + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/smithy-client/-/smithy-client-3.358.0.tgz", + "integrity": "sha512-oqctxWb9yAqCh4ENwUkt9MC01l5uKoy+QCiSUUhQ76k7R3lyGOge9ycyRyoKl+oZWvEpnjZevXQFqEfGzkL7bA==", "requires": { - "@aws-sdk/middleware-stack": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/middleware-stack": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-stream": "3.358.0", + "@smithy/types": "^1.0.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/token-providers": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.241.0.tgz", - "integrity": "sha512-79okvuOS7V559OIL/RalIPG98wzmWxeFOChFnbEjn2pKOyGQ6FJRwLPYZaVRtNdAtnkBNgRpmFq9dX843QxhtQ==", - "requires": { - "@aws-sdk/client-sso-oidc": "3.241.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/shared-ini-file-loader": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.358.0.tgz", + "integrity": "sha512-vATKNCwNhCSo2LzvtkIzW9Yp2/aKNR032VPtIWlDtWGGFhkzGi4FPS0VTdfefxz4rqPWfBz53mh54d9xylsWVw==", + "requires": { + "@aws-sdk/client-sso-oidc": "3.358.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/shared-ini-file-loader": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/types": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.226.0.tgz", - "integrity": "sha512-MmmNHrWeO4man7wpOwrAhXlevqtOV9ZLcH4RhnG5LmRce0RFOApx24HoKENfFCcOyCm5LQBlsXCqi0dZWDWU0A==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.357.0.tgz", + "integrity": "sha512-/riCRaXg3p71BeWnShrai0y0QTdXcouPSM0Cn1olZbzTf7s71aLEewrc96qFrL70XhY4XvnxMpqQh+r43XIL3g==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/url-parser": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/url-parser/-/url-parser-3.226.0.tgz", - "integrity": "sha512-p5RLE0QWyP0OcTOLmFcLdVgUcUEzmEfmdrnOxyNzomcYb0p3vUagA5zfa1HVK2azsQJFBv28GfvMnba9bGhObg==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/url-parser/-/url-parser-3.357.0.tgz", + "integrity": "sha512-fAaU6cFsaAba01lCRsRJiYR/LfXvX2wudyEyutBVglE4dWSoSeu3QJNxImIzTBULfbiFhz59++NQ1JUVx88IVg==", "requires": { - "@aws-sdk/querystring-parser": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/querystring-parser": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-base64": { - "version": "3.208.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-base64/-/util-base64-3.208.0.tgz", - "integrity": "sha512-PQniZph5A6N7uuEOQi+1hnMz/FSOK/8kMFyFO+4DgA1dZ5pcKcn5wiFwHkcTb/BsgVqQa3Jx0VHNnvhlS8JyTg==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-base64/-/util-base64-3.310.0.tgz", + "integrity": "sha512-v3+HBKQvqgdzcbL+pFswlx5HQsd9L6ZTlyPVL2LS9nNXnCcR3XgGz9jRskikRUuUvUXtkSG1J88GAOnJ/apTPg==", "requires": { - "@aws-sdk/util-buffer-from": "3.208.0", - "tslib": "^2.3.1" + "@aws-sdk/util-buffer-from": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-body-length-browser": { - "version": "3.188.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-body-length-browser/-/util-body-length-browser-3.188.0.tgz", - "integrity": "sha512-8VpnwFWXhnZ/iRSl9mTf+VKOX9wDE8QtN4bj9pBfxwf90H1X7E8T6NkiZD3k+HubYf2J94e7DbeHs7fuCPW5Qg==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-body-length-browser/-/util-body-length-browser-3.310.0.tgz", + "integrity": "sha512-sxsC3lPBGfpHtNTUoGXMQXLwjmR0zVpx0rSvzTPAuoVILVsp5AU/w5FphNPxD5OVIjNbZv9KsKTuvNTiZjDp9g==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-body-length-node": { - "version": "3.208.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-body-length-node/-/util-body-length-node-3.208.0.tgz", - "integrity": "sha512-3zj50e5g7t/MQf53SsuuSf0hEELzMtD8RX8C76f12OSRo2Bca4FLLYHe0TZbxcfQHom8/hOaeZEyTyMogMglqg==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-body-length-node/-/util-body-length-node-3.310.0.tgz", + "integrity": "sha512-2tqGXdyKhyA6w4zz7UPoS8Ip+7sayOg9BwHNidiGm2ikbDxm1YrCfYXvCBdwaJxa4hJfRVz+aL9e+d3GqPI9pQ==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-buffer-from": { - "version": "3.208.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-buffer-from/-/util-buffer-from-3.208.0.tgz", - "integrity": "sha512-7L0XUixNEFcLUGPeBF35enCvB9Xl+K6SQsmbrPk1P3mlV9mguWSDQqbOBwY1Ir0OVbD6H/ZOQU7hI/9RtRI0Zw==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-buffer-from/-/util-buffer-from-3.310.0.tgz", + "integrity": "sha512-i6LVeXFtGih5Zs8enLrt+ExXY92QV25jtEnTKHsmlFqFAuL3VBeod6boeMXkN2p9lbSVVQ1sAOOYZOHYbYkntw==", "requires": { - "@aws-sdk/is-array-buffer": "3.201.0", - "tslib": "^2.3.1" + "@aws-sdk/is-array-buffer": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-config-provider": { - "version": "3.208.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-config-provider/-/util-config-provider-3.208.0.tgz", - "integrity": "sha512-DSRqwrERUsT34ug+anlMBIFooBEGwM8GejC7q00Y/9IPrQy50KnG5PW2NiTjuLKNi7pdEOlwTSEocJE15eDZIg==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-config-provider/-/util-config-provider-3.310.0.tgz", + "integrity": "sha512-xIBaYo8dwiojCw8vnUcIL4Z5tyfb1v3yjqyJKJWV/dqKUFOOS0U591plmXbM+M/QkXyML3ypon1f8+BoaDExrg==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-defaults-mode-browser": { - "version": "3.234.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-defaults-mode-browser/-/util-defaults-mode-browser-3.234.0.tgz", - "integrity": "sha512-IHMKXjTbOD8XMz5+2oCOsVP94BYb9YyjXdns0aAXr2NAo7k2+RCzXQ2DebJXppGda1F6opFutoKwyVSN0cmbMw==", + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-defaults-mode-browser/-/util-defaults-mode-browser-3.358.0.tgz", + "integrity": "sha512-KGfw64wRL/gROLD4Gatda8cUsaNKNhSnx+yDDcG2WkFlFfLr6FHvTijpRxvIM2Jau2ZhcdGzbegLjsFxviTJAA==", "requires": { - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/types": "3.226.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", "bowser": "^2.11.0", - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-defaults-mode-node": { - "version": "3.234.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-defaults-mode-node/-/util-defaults-mode-node-3.234.0.tgz", - "integrity": "sha512-UGjQ+OjBYYhxFVtUY+jtr0ZZgzZh6OHtYwRhFt8IHewJXFCfZTyfsbX20szBj5y1S4HRIUJ7cwBLIytTqMbI5w==", - "requires": { - "@aws-sdk/config-resolver": "3.234.0", - "@aws-sdk/credential-provider-imds": "3.226.0", - "@aws-sdk/node-config-provider": "3.226.0", - "@aws-sdk/property-provider": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-defaults-mode-node/-/util-defaults-mode-node-3.358.0.tgz", + "integrity": "sha512-2C5on0yppDS0xGpFkHRqfrG9TeTq6ive1hPX1V8UCkiI/TBQYl88XCKCKct8zTcejyK9klZUDGI8QQTan2UWkw==", + "requires": { + "@aws-sdk/config-resolver": "3.357.0", + "@aws-sdk/credential-provider-imds": "3.357.0", + "@aws-sdk/node-config-provider": "3.357.0", + "@aws-sdk/property-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-endpoints": { - "version": "3.241.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.241.0.tgz", - "integrity": "sha512-jVf8bKrN22Ey0xLmj75sL7EUvm5HFpuOMkXsZkuXycKhCwLBcEUWlvtJYtRjOU1zScPQv9GMJd2QXQglp34iOQ==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.357.0.tgz", + "integrity": "sha512-XHKyS5JClT9su9hDif715jpZiWHQF9gKZXER8tW0gOizU3R9cyWc9EsJ2BRhFNhi7nt/JF/CLUEc5qDx3ETbUw==", "requires": { - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-hex-encoding": { - "version": "3.201.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-hex-encoding/-/util-hex-encoding-3.201.0.tgz", - "integrity": "sha512-7t1vR1pVxKx0motd3X9rI3m/xNp78p3sHtP5yo4NP4ARpxyJ0fokBomY8ScaH2D/B+U5o9ARxldJUdMqyBlJcA==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-hex-encoding/-/util-hex-encoding-3.310.0.tgz", + "integrity": "sha512-sVN7mcCCDSJ67pI1ZMtk84SKGqyix6/0A1Ab163YKn+lFBQRMKexleZzpYzNGxYzmQS6VanP/cfU7NiLQOaSfA==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-locate-window": { - "version": "3.208.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.208.0.tgz", - "integrity": "sha512-iua1A2+P7JJEDHVgvXrRJSvsnzG7stYSGQnBVphIUlemwl6nN5D+QrgbjECtrbxRz8asYFHSzhdhECqN+tFiBg==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.310.0.tgz", + "integrity": "sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-middleware": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-middleware/-/util-middleware-3.226.0.tgz", - "integrity": "sha512-B96CQnwX4gRvQdaQkdUtqvDPkrptV5+va6FVeJOocU/DbSYMAScLxtR3peMS8cnlOT6nL1Eoa42OI9AfZz1VwQ==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-middleware/-/util-middleware-3.357.0.tgz", + "integrity": "sha512-pV1krjZs7BdahZBfsCJMatE8kcor7GFsBOWrQgQDm9T0We5b5xPpOO2vxAD0RytBpY8Ky2ELs/+qXMv7l5fWIA==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-retry": { - "version": "3.229.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-retry/-/util-retry-3.229.0.tgz", - "integrity": "sha512-0zKTqi0P1inD0LzIMuXRIYYQ/8c1lWMg/cfiqUcIAF1TpatlpZuN7umU0ierpBFud7S+zDgg0oemh+Nj8xliJw==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-retry/-/util-retry-3.357.0.tgz", + "integrity": "sha512-SUqYJE9msbuOVq+vnUy+t0LH7XuYNFz66dSF8q6tedsbJK4j8tgya0I1Ct3m06ynGrXDJMaj39I7AXCyW9bjtw==", "requires": { - "@aws-sdk/service-error-classification": "3.229.0", - "tslib": "^2.3.1" + "@aws-sdk/service-error-classification": "3.357.0", + "tslib": "^2.5.0" + }, + "dependencies": { + "tslib": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" + } + } + }, + "@aws-sdk/util-stream": { + "version": "3.358.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-stream/-/util-stream-3.358.0.tgz", + "integrity": "sha512-zUhpjxAXV2+0eALlTU6uXRYMs10XYpcYzl3NtLRe4wWgnrOOOZnF/t5LQDoKXOfaMdzwZ+i90+PYr+6JQ58+7g==", + "requires": { + "@aws-sdk/fetch-http-handler": "3.357.0", + "@aws-sdk/node-http-handler": "3.357.0", + "@aws-sdk/types": "3.357.0", + "@aws-sdk/util-base64": "3.310.0", + "@aws-sdk/util-buffer-from": "3.310.0", + "@aws-sdk/util-hex-encoding": "3.310.0", + "@aws-sdk/util-utf8": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-uri-escape": { - "version": "3.201.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-uri-escape/-/util-uri-escape-3.201.0.tgz", - "integrity": "sha512-TeTWbGx4LU2c5rx0obHeDFeO9HvwYwQtMh1yniBz00pQb6Qt6YVOETVQikRZ+XRQwEyCg/dA375UplIpiy54mA==", + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-uri-escape/-/util-uri-escape-3.310.0.tgz", + "integrity": "sha512-drzt+aB2qo2LgtDoiy/3sVG8w63cgLkqFIa2NFlGpUgHFWTXkqtbgf4L5QdjRGKWhmZsnqkbtL7vkSWEcYDJ4Q==", "requires": { - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-user-agent-browser": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.226.0.tgz", - "integrity": "sha512-PhBIu2h6sPJPcv2I7ELfFizdl5pNiL4LfxrasMCYXQkJvVnoXztHA1x+CQbXIdtZOIlpjC+6BjDcE0uhnpvfcA==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.357.0.tgz", + "integrity": "sha512-JHaWlNIUkPNvXkqeDOrqFzAlAgdwZK5mZw7FQnCRvf8tdSogpGZSkuyb9Z6rLD9gC40Srbc2nepO1cFpeMsDkA==", "requires": { - "@aws-sdk/types": "3.226.0", + "@aws-sdk/types": "3.357.0", "bowser": "^2.11.0", - "tslib": "^2.3.1" + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, "@aws-sdk/util-user-agent-node": { - "version": "3.226.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.226.0.tgz", - "integrity": "sha512-othPc5Dz/pkYkxH+nZPhc1Al0HndQT8zHD4e9h+EZ+8lkd8n+IsnLfTS/mSJWrfiC6UlNRVw55cItstmJyMe/A==", + "version": "3.357.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.357.0.tgz", + "integrity": "sha512-RdpQoaJWQvcS99TVgSbT451iGrlH4qpWUWFA9U1IRhxOSsmC1hz8ME7xc8nci9SREx/ZlfT3ai6LpoAzAtIEMA==", "requires": { - "@aws-sdk/node-config-provider": "3.226.0", - "@aws-sdk/types": "3.226.0", - "tslib": "^2.3.1" + "@aws-sdk/node-config-provider": "3.357.0", + "@aws-sdk/types": "3.357.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, - "@aws-sdk/util-utf8-browser": { - "version": "3.188.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.188.0.tgz", - "integrity": "sha512-jt627x0+jE+Ydr9NwkFstg3cUvgWh56qdaqAMDsqgRlKD21md/6G226z/Qxl7lb1VEW2LlmCx43ai/37Qwcj2Q==", + "@aws-sdk/util-utf8": { + "version": "3.310.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8/-/util-utf8-3.310.0.tgz", + "integrity": "sha512-DnLfFT8uCO22uOJc0pt0DsSNau1GTisngBCDw8jQuWT5CqogMJu4b/uXmwEqfj8B3GX6Xsz8zOd6JpRlPftQoA==", "requires": { - "tslib": "^2.3.1" + "@aws-sdk/util-buffer-from": "3.310.0", + "tslib": "^2.5.0" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, - "@aws-sdk/util-utf8-node": { - "version": "3.208.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-node/-/util-utf8-node-3.208.0.tgz", - "integrity": "sha512-jKY87Acv0yWBdFxx6bveagy5FYjz+dtV8IPT7ay1E2WPWH1czoIdMAkc8tSInK31T6CRnHWkLZ1qYwCbgRfERQ==", + "@aws-sdk/util-utf8-browser": { + "version": "3.259.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", + "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", "requires": { - "@aws-sdk/util-buffer-from": "3.208.0", "tslib": "^2.3.1" }, "dependencies": { "tslib": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", - "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" } } }, @@ -1326,24 +1381,38 @@ } } }, - "@babel/helper-function-name": { - "version": "7.9.5", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.9.5.tgz", - "integrity": "sha512-JVcQZeXM59Cd1qanDUxv9fgJpt3NeKUaqBqUEvfmQ+BCOKq2xUgaWZW2hr0dkbyJgezYuplEoh5knmrnS68efw==", - "dev": true, - "requires": { - "@babel/helper-get-function-arity": "^7.8.3", - "@babel/template": "^7.8.3", - "@babel/types": "^7.9.5" - } + "@babel/helper-environment-visitor": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", + "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", + "dev": true }, - "@babel/helper-get-function-arity": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.8.3.tgz", - "integrity": "sha512-FVDR+Gd9iLjUMY1fzE2SR0IuaJToR4RkCDARVfsBBPSP53GEqSFjD8gNyxg246VUyc/ALRxFaAK8rVG7UT7xRA==", + "@babel/helper-hoist-variables": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", + "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", "dev": true, "requires": { - "@babel/types": "^7.8.3" + "@babel/types": "^7.22.5" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "dev": true + }, + "@babel/types": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", + "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", + "dev": true, + "requires": { + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.20", + "to-fast-properties": "^2.0.0" + } + } } }, "@babel/helper-member-expression-to-functions": { @@ -1425,6 +1494,12 @@ "@babel/types": "^7.8.3" } }, + "@babel/helper-string-parser": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", + "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", + "dev": true + }, "@babel/helper-validator-identifier": { "version": "7.9.5", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.9.5.tgz", @@ -1561,29 +1636,116 @@ } }, "@babel/traverse": { - "version": "7.9.5", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.9.5.tgz", - "integrity": "sha512-c4gH3jsvSuGUezlP6rzSJ6jf8fYjLj3hsMZRx/nX0h+fmHN0w+ekubRrHPqnMec0meycA2nwCsJ7dC8IPem2FQ==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.8.3", - "@babel/generator": "^7.9.5", - "@babel/helper-function-name": "^7.9.5", - "@babel/helper-split-export-declaration": "^7.8.3", - "@babel/parser": "^7.9.0", - "@babel/types": "^7.9.5", + "version": "7.23.2", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", + "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.22.13", + "@babel/generator": "^7.23.0", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-function-name": "^7.23.0", + "@babel/helper-hoist-variables": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/parser": "^7.23.0", + "@babel/types": "^7.23.0", "debug": "^4.1.0", - "globals": "^11.1.0", - "lodash": "^4.17.13" + "globals": "^11.1.0" }, "dependencies": { + "@babel/code-frame": { + "version": "7.22.13", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", + "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", + "dev": true, + "requires": { + "@babel/highlight": "^7.22.13", + "chalk": "^2.4.2" + } + }, + "@babel/generator": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", + "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", + "dev": true, + "requires": { + "@babel/types": "^7.23.0", + "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", + "jsesc": "^2.5.1" + } + }, + "@babel/helper-function-name": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", + "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", + "dev": true, + "requires": { + "@babel/template": "^7.22.15", + "@babel/types": "^7.23.0" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.22.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", + "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", + "dev": true, + "requires": { + "@babel/types": "^7.22.5" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "dev": true + }, + "@babel/highlight": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", + "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.22.20", + "chalk": "^2.4.2", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", + "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", + "dev": true + }, + "@babel/template": { + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", + "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.22.13", + "@babel/parser": "^7.22.15", + "@babel/types": "^7.22.15" + } + }, + "@babel/types": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", + "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", + "dev": true, + "requires": { + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.20", + "to-fast-properties": "^2.0.0" + } + }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } } } @@ -2469,6 +2631,45 @@ "@types/yargs": "^13.0.0" } }, + "@jridgewell/gen-mapping": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", + "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", + "dev": true + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.19", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", + "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "@kubernetes/client-node": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/@kubernetes/client-node/-/client-node-0.8.2.tgz", @@ -2506,6 +2707,37 @@ "type-detect": "4.0.8" } }, + "@smithy/protocol-http": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-1.1.0.tgz", + "integrity": "sha512-H5y/kZOqfJSqRkwtcAoVbqONmhdXwSgYNJ1Glk5Ry8qlhVVy5qUzD9EklaCH8/XLnoCsLO/F/Giee8MIvaBRkg==", + "requires": { + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "dependencies": { + "tslib": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" + } + } + }, + "@smithy/types": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-1.1.0.tgz", + "integrity": "sha512-KzmvisMmuwD2jZXuC9e65JrgsZM97y5NpDU7g347oB+Q+xQLU6hQZ5zFNNbEfwwOJHoOvEVTna+dk1h/lW7alw==", + "requires": { + "tslib": "^2.5.0" + }, + "dependencies": { + "tslib": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz", + "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==" + } + } + }, "@types/babel__core": { "version": "7.1.7", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.7.tgz", @@ -4408,9 +4640,9 @@ "integrity": "sha512-R9bHCvweUxxwkDwhjav5vxpFvdPGlVngtqmx4pIZfSUhM/Q4NiIUHB456BAf+Q1Nwu3HEZYONtu+Rya+af4jiQ==" }, "fast-xml-parser": { - "version": "4.0.11", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.11.tgz", - "integrity": "sha512-4aUg3aNRR/WjQAcpceODG1C3x3lFANXRo8+1biqfieHmg9pyMt7qB4lQV/Ta6sJCTbA5vfD8fnA8S54JATiFUA==", + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", + "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", "requires": { "strnum": "^1.0.5" } @@ -5156,9 +5388,9 @@ }, "dependencies": { "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true } } @@ -5190,9 +5422,9 @@ } }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true }, "supports-color": { @@ -5978,9 +6210,9 @@ "dev": true }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true }, "supports-color": { @@ -7230,9 +7462,9 @@ } }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true }, "supports-color": { @@ -7330,9 +7562,9 @@ } }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true }, "supports-color": { @@ -8078,9 +8310,9 @@ }, "dependencies": { "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "optional": true }, @@ -8748,9 +8980,9 @@ } }, "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==" }, "send": { "version": "0.17.2", @@ -9771,9 +10003,9 @@ "dev": true }, "word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", + "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==" }, "wrap-ansi": { "version": "6.2.0", diff --git a/frontend/server/package.json b/frontend/server/package.json index 7bac63c809..8395b2c49c 100644 --- a/frontend/server/package.json +++ b/frontend/server/package.json @@ -2,7 +2,7 @@ "description": "Frontend webserver package for Kubeflow Pipelines", "main": "server.js", "dependencies": { - "@aws-sdk/credential-providers": "^3.241.0", + "@aws-sdk/credential-providers": "^3.359.0", "@google-cloud/storage": "^2.5.0", "@kubernetes/client-node": "^0.8.2", "axios": ">=0.21.1", diff --git a/frontend/src/components/PipelinesDialog.tsx b/frontend/src/components/PipelinesDialog.tsx index f56fe8a625..992bbb4474 100644 --- a/frontend/src/components/PipelinesDialog.tsx +++ b/frontend/src/components/PipelinesDialog.tsx @@ -72,7 +72,8 @@ const PipelinesDialog: React.FC = (props): JSX.Element | n columns={props.pipelineSelectorColumns} emptyMessage='No pipelines found. Upload a pipeline and then try again.' initialSortColumn={PipelineSortKeys.CREATED_AT} - selectionChanged={(selectedPipeline: ApiPipeline) => { + selectionChanged={async (selectedId: string) => { + const selectedPipeline = await Apis.pipelineServiceApi.getPipeline(selectedId); setUnconfirmedSelectedPipeline(selectedPipeline); }} /> diff --git a/frontend/src/components/PipelinesDialogV2.tsx b/frontend/src/components/PipelinesDialogV2.tsx index dd15074748..be87860a57 100644 --- a/frontend/src/components/PipelinesDialogV2.tsx +++ b/frontend/src/components/PipelinesDialogV2.tsx @@ -21,7 +21,7 @@ import DialogActions from '@material-ui/core/DialogActions'; import { classes } from 'typestyle'; import { padding, commonCss } from 'src/Css'; import DialogContent from '@material-ui/core/DialogContent'; -import ResourceSelectorV2 from 'src/pages/ResourceSelectorV2'; +import ResourceSelector from 'src/pages/ResourceSelector'; import { Apis, PipelineSortKeys } from 'src/lib/Apis'; import { Column } from './CustomTable'; import { V2beta1Pipeline } from 'src/apisv2beta1/pipeline'; @@ -56,7 +56,7 @@ const PipelinesDialogV2: React.FC = (props): JSX.Element function getPipelinesList(): JSX.Element { return ( - - + )} diff --git a/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx b/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx index 4bc5b84b11..d0948760c7 100644 --- a/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx +++ b/frontend/src/components/tabs/StaticNodeDetailsV2.test.tsx @@ -76,12 +76,12 @@ describe('StaticNodeDetailsV2', () => { expect(screen.getAllByText('STRING').length).toEqual(2); screen.getByText('Image'); - screen.getByText('python:3.7'); + screen.getByText('python:3.9'); screen.getByText('Command'); expect(screen.getAllByText('sh').length).toEqual(2); // The yaml file we used in this test has command as follow: - /* + /* sh -c @@ -131,7 +131,7 @@ describe('StaticNodeDetailsV2', () => { expect(screen.getAllByText('STRING').length).toEqual(1); screen.getByText('Image'); - screen.getByText('python:3.7'); + screen.getByText('python:3.9'); screen.getByText('Command'); expect(screen.getAllByText('sh').length).toEqual(2); diff --git a/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml b/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml index f92719a2f5..fc07ad77ad 100644 --- a/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml +++ b/frontend/src/data/test/create_mount_delete_dynamic_pvc.yaml @@ -76,7 +76,7 @@ deploymentSpec: \ *\n\ndef consumer() -> str:\n with open('/data/file.txt', 'r') as file:\n\ \ content = file.read()\n print(content)\n return content\n\ \n" - image: python:3.7 + image: python:3.9 exec-createpvc: container: image: argostub/createpvc @@ -111,7 +111,7 @@ deploymentSpec: \ file.write('Hello world')\n with open('/data/file.txt', 'r')\ \ as file:\n content = file.read()\n print(content)\n return\ \ content\n\n" - image: python:3.7 + image: python:3.9 pipelineInfo: name: my-pipeline root: diff --git a/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml b/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml index 3e569eb2c5..c32ada3c00 100644 --- a/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml +++ b/frontend/src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml @@ -6,7 +6,7 @@ deploymentSpec: executors: exec-preprocess: container: - image: python:3.7 + image: python:3.9 args: - --executor_input - '{{$}}' @@ -105,7 +105,7 @@ deploymentSpec: \ Model artifact, which has a .metadata dictionary\n # to store arbitrary\ \ metadata for the output artifact.\n model.metadata['accuracy'] = 0.9\n\ \n" - image: python:3.7 + image: python:3.9 components: comp-preprocess: inputDefinitions: @@ -233,4 +233,4 @@ root: enableCache: true componentRef: name: comp-preprocess -defaultPipelineRoot: dummy_root +defaultPipelineRoot: minio://dummy_root diff --git a/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml b/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml index 1d79405a07..f12bc94a80 100644 --- a/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml +++ b/frontend/src/data/test/pipeline_with_loops_and_conditions.yaml @@ -588,7 +588,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.7 + image: python:3.9 exec-args-generator-op-2: container: args: @@ -617,7 +617,7 @@ deploymentSpec: \ 'A_a': '1',\n 'B_b': ['2', '20'],\n },\n \ \ {\n 'A_a': '10',\n 'B_b': ['22', '222'],\n \ \ },\n ]\n\n" - image: python:3.7 + image: python:3.9 exec-flip-coin-op: container: args: @@ -645,7 +645,7 @@ deploymentSpec: \ *\n\ndef flip_coin_op() -> str:\n \"\"\"Flip a coin and output heads\ \ or tails randomly.\"\"\"\n import random\n result = 'heads' if random.randint(0,\ \ 1) == 0 else 'tails'\n return result\n\n" - image: python:3.7 + image: python:3.9 exec-print-struct: container: args: @@ -671,7 +671,7 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_struct(struct: dict):\n print(struct)\n\n" - image: python:3.7 + image: python:3.9 exec-print-text: container: args: @@ -698,7 +698,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-2: container: args: @@ -725,7 +725,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-3: container: args: @@ -752,7 +752,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-4: container: args: @@ -779,7 +779,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-5: container: args: @@ -806,7 +806,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-6: container: args: @@ -833,7 +833,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-7: container: args: @@ -860,7 +860,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-8: container: args: @@ -887,7 +887,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 exec-print-text-9: container: args: @@ -914,7 +914,7 @@ deploymentSpec: - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef print_text(msg: str, msg2: Optional[str] = None):\n print(f'msg:\ \ {msg}, msg2: {msg2}')\n\n" - image: python:3.7 + image: python:3.9 pipelineInfo: name: pipeline-with-loops-and-conditions-multi-layers root: diff --git a/frontend/src/data/test/xgboost_sample_pipeline.yaml b/frontend/src/data/test/xgboost_sample_pipeline.yaml index 982676b2de..e7a5410417 100644 --- a/frontend/src/data/test/xgboost_sample_pipeline.yaml +++ b/frontend/src/data/test/xgboost_sample_pipeline.yaml @@ -198,7 +198,7 @@ components: artifactType: schemaTitle: system.Artifact schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root +defaultPipelineRoot: minio://dummy_root deploymentSpec: executors: exec-chicago-taxi-trips-dataset: @@ -253,7 +253,7 @@ deploymentSpec: \ try:\n os.makedirs(os.path.dirname(output_file))\n except\ \ OSError:\n pass\n with open(output_file, 'w') as f:\n \ \ f.write(_output_serializers[idx](_outputs[idx]))\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict: container: args: @@ -304,7 +304,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict-2: container: args: @@ -358,7 +358,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict-3: container: args: @@ -412,7 +412,7 @@ deploymentSpec: predictions_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-predict-4: container: args: @@ -463,7 +463,7 @@ deploymentSpec: _parser.add_argument(\"--predictions\", dest=\"predictions_path\", type=_make_parent_dirs_and_return_path,\ \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_predict(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-train: container: args: @@ -571,7 +571,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 exec-xgboost-train-2: container: args: @@ -679,7 +679,7 @@ deploymentSpec: , dest=\"model_config_path\", type=_make_parent_dirs_and_return_path, required=True,\ \ default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ \n_outputs = xgboost_train(**_parsed_args)\n" - image: python:3.7 + image: python:3.9 pipelineInfo: name: xgboost-sample-pipeline root: diff --git a/frontend/src/features.ts b/frontend/src/features.ts index a9e41f3567..769195139f 100644 --- a/frontend/src/features.ts +++ b/frontend/src/features.ts @@ -7,6 +7,9 @@ export interface Feature { export enum FeatureKey { V2 = 'v2', // Please start using V2_ALPHA instead of V2, because we have switched to V2_ALPHA as V2 feature is enabled by default. V2_ALPHA = 'v2_alpha', + FUNCTIONAL_COMPONENT = 'functional_component', + // We plan to refactor the class component to functional component. + // To avoid breacking current behavior, enable this feature to do the bugbash / validation test for functional components. } const FEATURE_V2 = { @@ -21,7 +24,13 @@ const FEATURE_V2_ALPHA = { active: true, }; -const features: Feature[] = [FEATURE_V2, FEATURE_V2_ALPHA]; +const FEATURE_FUNCTIONAL_COMPONENT = { + name: FeatureKey.FUNCTIONAL_COMPONENT, + description: 'Use functional component', + active: false, +}; + +const features: Feature[] = [FEATURE_V2, FEATURE_V2_ALPHA, FEATURE_FUNCTIONAL_COMPONENT]; declare global { var __FEATURE_FLAGS__: string; diff --git a/frontend/src/lib/v2/DynamicFlow.test.ts b/frontend/src/lib/v2/DynamicFlow.test.ts index c35f38de95..feaad8f386 100644 --- a/frontend/src/lib/v2/DynamicFlow.test.ts +++ b/frontend/src/lib/v2/DynamicFlow.test.ts @@ -142,6 +142,25 @@ describe('DynamicFlow', () => { expect(nodeMlmdInfo).toEqual({ execution }); }); + it('execution found with custom name', () => { + const label = 'custom-label'; + const elem: Node = { + id: 'task.exec', + data: { + label: label, + mlmdId: 1, + }, + type: NodeTypeNames.EXECUTION, + position: { x: 1, y: 2 }, + }; + + const execution = new Execution(); + execution.setId(1); + execution.getCustomPropertiesMap().set(TASK_NAME_KEY, new Value().setStringValue(label)); + const nodeMlmdInfo = getNodeMlmdInfo(elem, [execution], [], []); + expect(nodeMlmdInfo).toEqual({ execution }); + }); + it('artifact not exist', () => { const elem: Node = { id: 'artifact.exec.arti', diff --git a/frontend/src/lib/v2/DynamicFlow.ts b/frontend/src/lib/v2/DynamicFlow.ts index 9c2736b687..fc44ccd199 100644 --- a/frontend/src/lib/v2/DynamicFlow.ts +++ b/frontend/src/lib/v2/DynamicFlow.ts @@ -263,12 +263,14 @@ export function updateFlowElementsState( } return flowGraph; } - for (let elem of elems) { let updatedElem = Object.assign({}, elem); if (NodeTypeNames.EXECUTION === elem.type) { - const taskName = getTaskKeyFromNodeKey(elem.id); - const executions = getExecutionsUnderDAG(taskNameToExecution, taskName, executionLayers); + const executions = getExecutionsUnderDAG( + taskNameToExecution, + getTaskLabelByPipelineFlowElement(elem), + executionLayers, + ); if (executions) { (updatedElem.data as ExecutionFlowElementData).state = executions[0]?.getLastKnownState(); (updatedElem.data as ExecutionFlowElementData).mlmdId = executions[0]?.getId(); @@ -291,8 +293,11 @@ export function updateFlowElementsState( (updatedElem.data as ArtifactFlowElementData).mlmdId = linkedArtifact?.artifact?.getId(); } else if (NodeTypeNames.SUB_DAG === elem.type) { // TODO: Update sub-dag state based on future design. - const taskName = getTaskKeyFromNodeKey(elem.id); - const executions = getExecutionsUnderDAG(taskNameToExecution, taskName, executionLayers); + const executions = getExecutionsUnderDAG( + taskNameToExecution, + getTaskLabelByPipelineFlowElement(elem), + executionLayers, + ); if (executions) { (updatedElem.data as SubDagFlowElementData).state = executions[0]?.getLastKnownState(); (updatedElem.data as SubDagFlowElementData).mlmdId = executions[0]?.getId(); @@ -303,6 +308,12 @@ export function updateFlowElementsState( return flowGraph; } +function getTaskLabelByPipelineFlowElement(elem: PipelineFlowElement) { + const taskLabel = elem.data?.label; + if (taskLabel === undefined) return getTaskKeyFromNodeKey(elem.id); + return taskLabel; +} + function getExecutionsUnderDAG( taskNameToExecution: Map, taskName: string, @@ -337,9 +348,9 @@ export function getNodeMlmdInfo( ); if (NodeTypeNames.EXECUTION === elem.type) { - const taskName = getTaskKeyFromNodeKey(elem.id); + const taskLabel = getTaskLabelByPipelineFlowElement(elem); const executions = taskNameToExecution - .get(taskName) + .get(taskLabel) ?.filter(exec => exec.getId() === elem.data?.mlmdId); return executions ? { execution: executions[0] } : {}; } else if (NodeTypeNames.ARTIFACT === elem.type) { @@ -361,9 +372,9 @@ export function getNodeMlmdInfo( return { execution, linkedArtifact }; } else if (NodeTypeNames.SUB_DAG === elem.type) { // TODO: Update sub-dag state based on future design. - const taskName = getTaskKeyFromNodeKey(elem.id); + const taskLabel = getTaskLabelByPipelineFlowElement(elem); const executions = taskNameToExecution - .get(taskName) + .get(taskLabel) ?.filter(exec => exec.getId() === elem.data?.mlmdId); return executions ? { execution: executions[0] } : {}; } diff --git a/frontend/src/lib/v2/WorkflowUtils.test.ts b/frontend/src/lib/v2/WorkflowUtils.test.ts index ee50388f34..d3d5f38263 100644 --- a/frontend/src/lib/v2/WorkflowUtils.test.ts +++ b/frontend/src/lib/v2/WorkflowUtils.test.ts @@ -143,7 +143,7 @@ PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-scr "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef preprocess(\n # An input parameter of type string.\n message: str,\n # An input parameter of type dict.\n input_dict_parameter: Dict[str, int],\n # An input parameter of type list.\n input_list_parameter: List[str],\n # Use Output[T] to get a metadata-rich handle to the output artifact\n # of type `Dataset`.\n output_dataset_one: Output[Dataset],\n # A locally accessible filepath for another output artifact of type\n # `Dataset`.\n output_dataset_two_path: OutputPath('Dataset'),\n # A locally accessible filepath for an output parameter of type string.\n output_parameter_path: OutputPath(str),\n # A locally accessible filepath for an output parameter of type bool.\n output_bool_parameter_path: OutputPath(bool),\n # A locally accessible filepath for an output parameter of type dict.\n output_dict_parameter_path: OutputPath(Dict[str, int]),\n # A locally accessible filepath for an output parameter of type list.\n output_list_parameter_path: OutputPath(List[str]),\n):\n \"\"\"Dummy preprocessing step.\"\"\"\n\n # Use Dataset.path to access a local file path for writing.\n # One can also use Dataset.uri to access the actual URI file path.\n with open(output_dataset_one.path, 'w') as f:\n f.write(message)\n\n # OutputPath is used to just pass the local file path of the output artifact\n # to the function.\n with open(output_dataset_two_path, 'w') as f:\n f.write(message)\n\n with open(output_parameter_path, 'w') as f:\n f.write(message)\n\n with open(output_bool_parameter_path, 'w') as f:\n f.write(\n str(True)) # use either `str()` or `json.dumps()` for bool values.\n\n import json\n with open(output_dict_parameter_path, 'w') as f:\n f.write(json.dumps(input_dict_parameter))\n\n with open(output_list_parameter_path, 'w') as f:\n f.write(json.dumps(input_list_parameter))\n\n", ], env: [], - image: 'python:3.7', + image: 'python:3.9', lifecycle: undefined, resources: undefined, }); @@ -174,7 +174,7 @@ PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-scr "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef producer() -> str:\n with open('/data/file.txt', 'w') as file:\n file.write('Hello world')\n with open('/data/file.txt', 'r') as file:\n content = file.read()\n print(content)\n return content\n\n", ], env: [], - image: 'python:3.7', + image: 'python:3.9', lifecycle: undefined, resources: undefined, }); diff --git a/frontend/src/mlmd/Utils.tsx b/frontend/src/mlmd/Utils.tsx index 683f162c14..334d7e1f77 100644 --- a/frontend/src/mlmd/Utils.tsx +++ b/frontend/src/mlmd/Utils.tsx @@ -116,4 +116,5 @@ export function getMetadataValue(value?: Value): string | number | Struct | unde case Value.ValueCase.VALUE_NOT_SET: return ''; } + return ''; } diff --git a/frontend/src/pages/ExecutionDetails.tsx b/frontend/src/pages/ExecutionDetails.tsx index de896b69ba..15bec58dd6 100644 --- a/frontend/src/pages/ExecutionDetails.tsx +++ b/frontend/src/pages/ExecutionDetails.tsx @@ -262,6 +262,7 @@ function parseEventsByType( [Event.Type.OUTPUT]: [], [Event.Type.INTERNAL_INPUT]: [], [Event.Type.INTERNAL_OUTPUT]: [], + [Event.Type.PENDING_OUTPUT]: [], }; if (!response) { diff --git a/frontend/src/pages/ExperimentDetails.tsx b/frontend/src/pages/ExperimentDetails.tsx index 85f500e8b9..29e3787a24 100644 --- a/frontend/src/pages/ExperimentDetails.tsx +++ b/frontend/src/pages/ExperimentDetails.tsx @@ -59,6 +59,7 @@ const css = stylesheet({ fontSize: 12, minHeight: 16, paddingLeft: 0, + marginRight: 0, }, cardContent: { color: color.secondaryText, @@ -90,7 +91,7 @@ const css = stylesheet({ color: '#0d652d', }, recurringRunsCard: { - width: 158, + width: 270, }, recurringRunsDialog: { minWidth: 600, @@ -170,7 +171,17 @@ export class ExperimentDetails extends Page<{}, ExperimentDetailsState> { elevation={0} >
-
Recurring run configs
+
+ Recurring run configs + +
{ > {activeRecurringRunsCount + ' active'}
-
= props => { const namespace = React.useContext(NamespaceContext); - return ; + return isFeatureEnabled(FeatureKey.FUNCTIONAL_COMPONENT) ? ( + + ) : ( + + ); }; export default EnhancedNewExperiment; diff --git a/frontend/src/pages/NewRun.test.tsx b/frontend/src/pages/NewRun.test.tsx index 7304b76c07..db00f398f7 100644 --- a/frontend/src/pages/NewRun.test.tsx +++ b/frontend/src/pages/NewRun.test.tsx @@ -590,6 +590,10 @@ describe('NewRun', () => { const expectedPipeline = await screen.findByText(newPipeline.name); fireEvent.click(expectedPipeline); + await waitFor(() => { + expect(getPipelineSpy).toHaveBeenCalled(); + }); + const usePipelineButton = screen.getByText('Use this pipeline'); fireEvent.click(usePipelineButton); @@ -774,6 +778,10 @@ describe('NewRun', () => { const expectedPipeline = await screen.findByText(newPipeline.name); fireEvent.click(expectedPipeline); + await waitFor(() => { + expect(getPipelineSpy).toHaveBeenCalled(); + }); + const usePipelineButton = screen.getByText('Use this pipeline'); fireEvent.click(usePipelineButton); @@ -790,6 +798,10 @@ describe('NewRun', () => { const expectedPipelineVersion = await screen.findByText(latestPipelineVersion.name); fireEvent.click(expectedPipelineVersion); + await waitFor(() => { + expect(getPipelineVersionSpy).toHaveBeenCalled(); + }); + const usePipelineVersionBtn = screen.getByText('Use this pipeline version'); fireEvent.click(usePipelineVersionBtn); @@ -1671,6 +1683,10 @@ describe('NewRun', () => { const expectedPipeline = await screen.findByText(pipelineWithParams.name); fireEvent.click(expectedPipeline); + await waitFor(() => { + expect(getPipelineSpy).toHaveBeenCalled(); + }); + const usePipelineButton = screen.getByText('Use this pipeline'); fireEvent.click(usePipelineButton); @@ -1683,7 +1699,7 @@ describe('NewRun', () => { await screen.findByDisplayValue('prefilled value 1'); await screen.findByLabelText('param-2'); await screen.findByDisplayValue('prefilled value 2'); - }); + }, 10000); it('trims whitespace from the pipeline params', async () => { tree = shallow(); @@ -1729,6 +1745,10 @@ describe('NewRun', () => { const expectedPipeline = await screen.findByText(pipelineWithParams.name); fireEvent.click(expectedPipeline); + await waitFor(() => { + expect(getPipelineSpy).toHaveBeenCalled(); + }); + const usePipelineButton = screen.getByText('Use this pipeline'); fireEvent.click(usePipelineButton); diff --git a/frontend/src/pages/NewRun.tsx b/frontend/src/pages/NewRun.tsx index c376f36286..3ac7aded21 100644 --- a/frontend/src/pages/NewRun.tsx +++ b/frontend/src/pages/NewRun.tsx @@ -348,9 +348,14 @@ export class NewRun extends Page { columns={this.pipelineVersionSelectorColumns} emptyMessage='No pipeline versions found. Select or upload a pipeline then try again.' initialSortColumn={PipelineVersionSortKeys.CREATED_AT} - selectionChanged={(selectedPipelineVersion: ApiPipelineVersion) => - this.setStateSafe({ unconfirmedSelectedPipelineVersion: selectedPipelineVersion }) - } + selectionChanged={async (selectedId: string) => { + const selectedPipelineVersion = await Apis.pipelineServiceApi.getPipelineVersion( + selectedId, + ); + this.setStateSafe({ + unconfirmedSelectedPipelineVersion: selectedPipelineVersion, + }); + }} toolbarActionMap={buttons .upload(() => this.setStateSafe({ @@ -432,9 +437,12 @@ export class NewRun extends Page { columns={this.experimentSelectorColumns} emptyMessage='No experiments found. Create an experiment and then try again.' initialSortColumn={ExperimentSortKeys.CREATED_AT} - selectionChanged={(selectedExperiment: ApiExperiment) => - this.setStateSafe({ unconfirmedSelectedExperiment: selectedExperiment }) - } + selectionChanged={async (selectedId: string) => { + const selectedExperiment = await Apis.experimentServiceApi.getExperiment( + selectedId, + ); + this.setStateSafe({ unconfirmedSelectedExperiment: selectedExperiment }); + }} /> @@ -458,6 +466,7 @@ export class NewRun extends Page { {/* Run metadata inputs */} { variant='outlined' /> { pipeline_id: ORIGINAL_TEST_PIPELINE_ID, pipeline_version_id: ORIGINAL_TEST_PIPELINE_VERSION_ID, }, - runtime_config: { parameters: {}, pipeline_root: 'dummy_root' }, + runtime_config: { parameters: {}, pipeline_root: 'minio://dummy_root' }, service_account: '', }), ); diff --git a/frontend/src/pages/NewRunV2.tsx b/frontend/src/pages/NewRunV2.tsx index f5cbc1e9b6..68d979e407 100644 --- a/frontend/src/pages/NewRunV2.tsx +++ b/frontend/src/pages/NewRunV2.tsx @@ -51,7 +51,7 @@ import { classes, stylesheet } from 'typestyle'; import { PageProps } from './Page'; import PipelinesDialogV2 from 'src/components/PipelinesDialogV2'; import { V2beta1RecurringRun, RecurringRunMode } from 'src/apisv2beta1/recurringrun'; -import ResourceSelectorV2 from 'src/pages/ResourceSelectorV2'; +import ResourceSelector from 'src/pages/ResourceSelector'; import { convertExperimentToResource, convertPipelineVersionToResource, @@ -823,7 +823,7 @@ function PipelineVersionSelector(props: PipelineVersionSelectorProps) { PaperProps={{ id: 'pipelineVersionSelectorDialog' }} > - - { const getV2RecurringRunSpy = jest.spyOn(Apis.recurringRunServiceApi, 'getRecurringRun'); const getExperimentSpy = jest.spyOn(Apis.experimentServiceApiV2, 'getExperiment'); const deletePipelineVersionSpy = jest.spyOn(Apis.pipelineServiceApiV2, 'deletePipelineVersion'); - const getPipelineVersionTemplateSpy = jest.spyOn( - Apis.pipelineServiceApi, - 'getPipelineVersionTemplate', - ); const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph'); + const PIPELINE_VERSION_ID = 'test-pipeline-version-id'; let tree: ShallowWrapper | ReactWrapper; let testV1Pipeline: ApiPipeline = {}; @@ -65,19 +62,23 @@ describe('PipelineDetails', () => { let testV1Run: ApiRunDetail = {}; let testV1RecurringRun: ApiJob = {}; let testV2Pipeline: V2beta1Pipeline = {}; - let testV2PipelineVersion: V2beta1PipelineVersion = {}; + let originalTestV2PipelineVersion: V2beta1PipelineVersion = {}; + let newTestV2PipelineVersion: V2beta1PipelineVersion = {}; let testV2Run: V2beta1Run = {}; let testV2RecurringRun: V2beta1RecurringRun = {}; - function generateProps(fromRunSpec = false, fromRecurringRunSpec = false): PageProps { + function generateProps( + versionId?: string, + fromRunSpec = false, + fromRecurringRunSpec = false, + ): PageProps { let params = {}; // If no fromXXX parameter is provided, it means KFP UI expects to // show Pipeline detail with pipeline version ID if (!fromRunSpec && !fromRecurringRunSpec) { params = { - [RouteParams.pipelineId]: testV1Pipeline.id, - [RouteParams.pipelineVersionId]: - (testV1Pipeline.default_version && testV1Pipeline.default_version!.id) || '', + [RouteParams.pipelineId]: testV2Pipeline.pipeline_id, + [RouteParams.pipelineVersionId]: versionId || '', }; } @@ -155,7 +156,7 @@ describe('PipelineDetails', () => { display_name: 'test pipeline', }; - testV2PipelineVersion = { + originalTestV2PipelineVersion = { display_name: 'test-pipeline-version', pipeline_id: 'test-pipeline-id', pipeline_version_id: 'test-pipeline-version-id', @@ -164,6 +165,15 @@ describe('PipelineDetails', () => { ), }; + newTestV2PipelineVersion = { + display_name: 'new-test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'new-test-pipeline-version-id', + pipeline_spec: JsYaml.safeLoad( + 'spec:\n arguments:\n parameters:\n - name: output\n', + ), + }; + testV2Run = { run_id: 'test-run-id', display_name: 'test run', @@ -185,9 +195,11 @@ describe('PipelineDetails', () => { getV1RecurringRunSpy.mockImplementation(() => Promise.resolve(testV1RecurringRun)); getV2PipelineSpy.mockImplementation(() => Promise.resolve(testV2Pipeline)); - getV2PipelineVersionSpy.mockImplementation(() => Promise.resolve(testV2PipelineVersion)); + getV2PipelineVersionSpy.mockImplementation(() => + Promise.resolve(originalTestV2PipelineVersion), + ); listV2PipelineVersionsSpy.mockImplementation(() => - Promise.resolve({ versions: [testV2PipelineVersion] }), + Promise.resolve({ pipeline_versions: [originalTestV2PipelineVersion] }), ); getV2RunSpy.mockImplementation(() => Promise.resolve(testV2Run)); getV2RecurringRunSpy.mockImplementation(() => Promise.resolve(testV2RecurringRun)); @@ -198,10 +210,6 @@ describe('PipelineDetails', () => { display_name: 'test experiment', } as V2beta1Experiment), ); - // getTemplateSpy.mockImplementation(() => Promise.resolve({ template: 'test template' })); - getPipelineVersionTemplateSpy.mockImplementation(() => - Promise.resolve({ template: 'test template' }), - ); createGraphSpy.mockImplementation(() => new graphlib.Graph()); }); @@ -214,12 +222,12 @@ describe('PipelineDetails', () => { it('shows pipeline name in page name, and breadcrumb to go back to pipelines', async () => { tree = shallow(); - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); expect(updateToolbarSpy).toHaveBeenLastCalledWith( expect.objectContaining({ breadcrumbs: [{ displayName: 'Pipelines', href: RoutePage.PIPELINES }], - pageTitle: testV1Pipeline.name + ' (' + testV1PipelineVersion.name + ')', + pageTitle: + testV2Pipeline.display_name + ' (' + originalTestV2PipelineVersion.display_name + ')', }), ); }); @@ -228,11 +236,10 @@ describe('PipelineDetails', () => { 'shows all runs breadcrumbs, and "Pipeline details" as page title when the pipeline ' + 'comes from a run spec that does not have an experiment', async () => { - tree = shallow(); + tree = shallow(); await getV1RunSpy; await getV2RunSpy; await createGraphSpy; - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); expect(updateToolbarSpy).toHaveBeenLastCalledWith( expect.objectContaining({ @@ -253,10 +260,9 @@ describe('PipelineDetails', () => { 'shows all runs breadcrumbs, and "Pipeline details" as page title when the pipeline ' + 'comes from a recurring run spec that does not have an experiment', async () => { - tree = shallow(); + tree = shallow(); await getV1RecurringRunSpy; await getV2RecurringRunSpy; - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); expect(updateToolbarSpy).toHaveBeenLastCalledWith( expect.objectContaining({ @@ -281,11 +287,10 @@ describe('PipelineDetails', () => { 'comes from a run spec that has an experiment', async () => { testV2Run.experiment_id = 'test-experiment-id'; - tree = shallow(); + tree = shallow(); await getV1RunSpy; await getV2RunSpy; await getExperimentSpy; - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); expect(updateToolbarSpy).toHaveBeenLastCalledWith( expect.objectContaining({ @@ -314,11 +319,10 @@ describe('PipelineDetails', () => { 'comes from a recurring run spec that has an experiment', async () => { testV2RecurringRun.experiment_id = 'test-experiment-id'; - tree = shallow(); + tree = shallow(); await getV1RecurringRunSpy; await getV2RecurringRunSpy; await getExperimentSpy; - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); expect(updateToolbarSpy).toHaveBeenLastCalledWith( expect.objectContaining({ @@ -354,7 +358,7 @@ describe('PipelineDetails', () => { workflow_manifest: '{"spec": {"arguments": {"parameters": [{"name": "output"}]}}}', }; - tree = shallow(); + tree = shallow(); await getV1RunSpy; await getV2RunSpy; await TestUtils.flushPromises(); @@ -368,10 +372,15 @@ describe('PipelineDetails', () => { it( 'directly use pipeline_manifest dumped from ' + 'pipeline_spec in run as template string (v2)', async () => { - jest.spyOn(features, 'isFeatureEnabled').mockReturnValue(true); + jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { + if (featureKey === features.FeatureKey.V2_ALPHA) { + return true; + } + return false; + }); testV2Run.pipeline_spec = { spec: { arguments: { parameters: [{ name: 'output' }] } } }; - tree = shallow(); + tree = shallow(); await getV1RunSpy; await getV2RunSpy; await TestUtils.flushPromises(); @@ -386,12 +395,17 @@ describe('PipelineDetails', () => { 'directly use pipeline_manifest dumped from pipeline_spec ' + 'in recurring run as template string (v2)', async () => { - jest.spyOn(features, 'isFeatureEnabled').mockReturnValue(true); + jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { + if (featureKey === features.FeatureKey.V2_ALPHA) { + return true; + } + return false; + }); testV2RecurringRun.pipeline_spec = { spec: { arguments: { parameters: [{ name: 'output' }] } }, }; - tree = shallow(); + tree = shallow(); await getV1RecurringRunSpy; await getV2RecurringRunSpy; await TestUtils.flushPromises(); @@ -403,15 +417,19 @@ describe('PipelineDetails', () => { ); it('use pipeline_version_id in run to get pipeline template string (v2)', async () => { - jest.spyOn(features, 'isFeatureEnabled').mockReturnValue(true); + jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { + if (featureKey === features.FeatureKey.V2_ALPHA) { + return true; + } + return false; + }); testV2Run.pipeline_version_reference.pipeline_id = 'test-pipeline-id'; testV2Run.pipeline_version_reference.pipeline_version_id = 'test-pipeline-version-id'; - tree = shallow(); + tree = shallow(); await getV1RunSpy; await getV2RunSpy; await getV2PipelineVersionSpy; - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); expect(tree.state('templateString')).toBe( @@ -419,9 +437,29 @@ describe('PipelineDetails', () => { ); }); + it('calls listPipelineVersions() if no pipeline version id', async () => { + listV2PipelineVersionsSpy.mockImplementation(() => + Promise.resolve({ + pipeline_versions: [newTestV2PipelineVersion, originalTestV2PipelineVersion], + }), + ); + render(); + + await waitFor(() => { + expect(listV2PipelineVersionsSpy).toHaveBeenCalled(); + }); + + expect(updateToolbarSpy).toHaveBeenLastCalledWith( + expect.objectContaining({ + breadcrumbs: [{ displayName: 'Pipelines', href: RoutePage.PIPELINES }], + pageTitle: testV2Pipeline.display_name + ' (' + newTestV2PipelineVersion.display_name + ')', + }), + ); + }); + it('renders "No graph to show" if it is empty pipeline', async () => { TestUtils.makeErrorResponse(getV2PipelineVersionSpy, 'No pipeline version is found'); - render(); + render(); await waitFor(() => { expect(getV2PipelineVersionSpy).toHaveBeenCalled(); @@ -431,15 +469,19 @@ describe('PipelineDetails', () => { }); it('use pipeline_version_id in recurring run to get pipeline template string (v2)', async () => { - jest.spyOn(features, 'isFeatureEnabled').mockReturnValue(true); + jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { + if (featureKey === features.FeatureKey.V2_ALPHA) { + return true; + } + return false; + }); testV2RecurringRun.pipeline_version_reference.pipeline_id = 'test-pipeline-id'; testV2RecurringRun.pipeline_version_reference.pipeline_version_id = 'test-pipeline-version-id'; - tree = shallow(); + tree = shallow(); await getV1RecurringRunSpy; await getV2RecurringRunSpy; await getV2PipelineVersionSpy; - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); expect(tree.state('templateString')).toBe( @@ -455,26 +497,28 @@ describe('PipelineDetails', () => { pipeline_id: 'run-pipeline-id', workflow_manifest: 'not valid JSON', }; + render(); - render(); await waitFor(() => { - expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error - expect(updateBannerSpy).toHaveBeenLastCalledWith( - expect.objectContaining({ - additionalInfo: 'Unexpected token o in JSON at position 1', - message: `Failed to parse pipeline spec from run with ID: ${ - testV1Run.run!.id - }. Click Details for more information.`, - mode: 'error', - }), - ); + expect(getV1RunSpy).toHaveBeenCalled(); }); + + expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error + expect(updateBannerSpy).toHaveBeenLastCalledWith( + expect.objectContaining({ + additionalInfo: 'Unexpected token o in JSON at position 1', + message: `Failed to parse pipeline spec from run with ID: ${ + testV1Run.run!.id + }. Click Details for more information.`, + mode: 'error', + }), + ); }, ); it('shows load error banner when failing to get run details, when loading from run spec', async () => { TestUtils.makeErrorResponseOnce(getV1RunSpy, 'woops'); - tree = shallow(); + tree = shallow(); await getV1PipelineSpy; await TestUtils.flushPromises(); expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error @@ -493,7 +537,7 @@ describe('PipelineDetails', () => { async () => { testV2Run.experiment_id = 'test-experiment-id'; TestUtils.makeErrorResponse(getExperimentSpy, 'woops'); - tree = shallow(); + tree = shallow(); await getV1PipelineSpy; await TestUtils.flushPromises(); expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error @@ -507,20 +551,6 @@ describe('PipelineDetails', () => { }, ); - it('uses an empty string and does not show error when getTemplate response is empty', async () => { - getPipelineVersionTemplateSpy.mockImplementationOnce(() => Promise.resolve({})); - - tree = shallow(); - await getV1PipelineSpy; - await TestUtils.flushPromises(); - - // No errors - expect(updateBannerSpy).toHaveBeenCalledTimes(1); // Once to clear banner - expect(updateBannerSpy).toHaveBeenLastCalledWith(expect.objectContaining({})); - - expect(tree.state('templateString')).toBe(''); - }); - it('shows load error banner when failing to get pipeline', async () => { TestUtils.makeErrorResponseOnce(getV1PipelineSpy, 'woops'); tree = shallow(); @@ -536,34 +566,113 @@ describe('PipelineDetails', () => { ); }); - it('shows load error banner when failing to get pipeline template', async () => { - TestUtils.makeErrorResponseOnce(getPipelineVersionTemplateSpy, 'woops'); - tree = shallow(); - await getV1PipelineSpy; - await TestUtils.flushPromises(); - expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error + it('shows load error banner when failing to get pipeline version', async () => { + TestUtils.makeErrorResponse(getV2PipelineVersionSpy, 'No pipeline version is found'); + render(); + + await waitFor(() => { + expect(getV2PipelineVersionSpy).toHaveBeenCalled(); + // get version error will use empty string as template string, which won't call createGraph() + expect(createGraphSpy).toHaveBeenCalledTimes(0); + }); + + expect(updateBannerSpy).toHaveBeenCalledTimes(2); // // Once to clear banner, once to show error expect(updateBannerSpy).toHaveBeenLastCalledWith( expect.objectContaining({ - additionalInfo: 'woops', - message: 'Cannot retrieve pipeline template. Click Details for more information.', + additionalInfo: 'No pipeline version is found', + message: 'Cannot retrieve pipeline version. Click Details for more information.', mode: 'error', }), ); }); + it( + 'uses an empty string and does not show error ' + + 'when pipeline_spec in the response of getPipelineVersion() is undefined', + async () => { + jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { + if (featureKey === features.FeatureKey.V2_ALPHA) { + return true; + } + return false; + }); + getV2PipelineVersionSpy.mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + pipeline_spec: undefined, // empty pipeline_spec + }); + render(); + + await waitFor(() => { + expect(getV2PipelineVersionSpy).toHaveBeenCalled(); + // empty template string from empty pipeline_spec and it won't call createGraph() + expect(createGraphSpy).toHaveBeenCalledTimes(0); + }); + + // No errors + expect(updateBannerSpy).toHaveBeenCalledTimes(1); // Once to clear banner + expect(updateBannerSpy).toHaveBeenLastCalledWith(expect.objectContaining({})); + }, + ); + + it( + 'shows no graph error banner ' + + 'when pipeline_spec in the response of getPipelineVersion() is invalid format', + async () => { + jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { + if (featureKey === features.FeatureKey.V2_ALPHA) { + return true; + } + return false; + }); + getV2PipelineVersionSpy.mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + pipeline_spec: {}, // invalid pipeline_spec + }); + render(); + + await waitFor(() => { + expect(getV2PipelineVersionSpy).toHaveBeenCalled(); + }); + + expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error + expect(updateBannerSpy).toHaveBeenLastCalledWith( + expect.objectContaining({ + additionalInfo: 'Important infomation is missing. Pipeline Spec is invalid.', + message: 'Error: failed to generate Pipeline graph. Click Details for more information.', + mode: 'error', + }), + ); + }, + ); + it('shows no graph error banner when failing to parse graph', async () => { - getPipelineVersionTemplateSpy.mockResolvedValue({ - template: ` - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: entry-point-test- - `, + jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { + if (featureKey === features.FeatureKey.V2_ALPHA) { + return true; + } + return false; + }); + getV2PipelineVersionSpy.mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + pipeline_spec: { + apiVersion: 'argoproj.io/v1alpha1', + kind: 'Workflow', + }, }); TestUtils.makeErrorResponse(createGraphSpy, 'bad graph'); - tree = shallow(); - await getPipelineVersionTemplateSpy; - await TestUtils.flushPromises(); + render(); + + await waitFor(() => { + expect(getV2PipelineVersionSpy).toHaveBeenCalled(); + expect(createGraphSpy).toHaveBeenCalled(); + }); + expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error expect(updateBannerSpy).toHaveBeenLastCalledWith( expect.objectContaining({ @@ -574,27 +683,8 @@ describe('PipelineDetails', () => { ); }); - it('clears the error banner when refreshing the page', async () => { - TestUtils.makeErrorResponseOnce(getPipelineVersionTemplateSpy, 'woops'); - tree = shallow(); - await TestUtils.flushPromises(); - - expect(updateBannerSpy).toHaveBeenLastCalledWith( - expect.objectContaining({ - additionalInfo: 'woops', - message: 'Cannot retrieve pipeline template. Click Details for more information.', - mode: 'error', - }), - ); - - (tree.instance() as PipelineDetails).refresh(); - - expect(updateBannerSpy).toHaveBeenLastCalledWith({}); - }); - it('has a new experiment button if it has a pipeline reference', async () => { tree = shallow(); - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; const newExperimentBtn = instance.getInitialToolbarState().actions[ButtonKeys.NEW_EXPERIMENT]; @@ -602,8 +692,7 @@ describe('PipelineDetails', () => { }); it("has 'clone run' toolbar button if viewing an embedded pipeline", async () => { - tree = shallow(); - await getPipelineVersionTemplateSpy; + tree = shallow(); await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; /* create run and create pipeline version, so 2 */ @@ -613,8 +702,7 @@ describe('PipelineDetails', () => { }); it("has 'clone recurring run' toolbar button if viewing an embedded pipeline from recurring run", async () => { - tree = shallow(); - await getPipelineVersionTemplateSpy; + tree = shallow(); await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; /* create run and create pipeline version, so 2 */ @@ -629,7 +717,7 @@ describe('PipelineDetails', () => { 'clicking clone run button when viewing embedded pipeline navigates to ' + 'the new run page (clone a run) with run ID', async () => { - tree = shallow(); + tree = shallow(); await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; const cloneRunBtn = instance.getInitialToolbarState().actions[ButtonKeys.CLONE_RUN]; @@ -645,7 +733,7 @@ describe('PipelineDetails', () => { 'clicking clone recurring run button when viewing embedded pipeline from recurring run' + 'navigates to the new run page (clone a recurring run) with recurring run ID', async () => { - tree = shallow(); + tree = shallow(); await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; const cloneRecurringRunBtn = instance.getInitialToolbarState().actions[ @@ -661,8 +749,7 @@ describe('PipelineDetails', () => { ); it("has 'create run' toolbar button if not viewing an embedded pipeline", async () => { - tree = shallow(); - await getPipelineVersionTemplateSpy; + tree = shallow(); await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; /* create run, create pipeline version, create experiment and delete run, so 4 */ @@ -673,8 +760,23 @@ describe('PipelineDetails', () => { expect(newRunBtn).toBeDefined(); }); + it('uses selected version ID to create run if URL does not contain version ID', async () => { + tree = shallow(); + await TestUtils.flushPromises(); + const instance = tree.instance() as PipelineDetails; + const newRunFromPipelineVersionBtn = instance.getInitialToolbarState().actions[ + ButtonKeys.NEW_RUN_FROM_PIPELINE_VERSION + ]; + newRunFromPipelineVersionBtn.action(); + expect(historyPushSpy).toHaveBeenCalledTimes(1); + expect(historyPushSpy).toHaveBeenLastCalledWith( + RoutePage.NEW_RUN + + `?${QUERY_PARAMS.pipelineId}=${testV2Pipeline.pipeline_id}&${QUERY_PARAMS.pipelineVersionId}=${originalTestV2PipelineVersion.pipeline_version_id}`, + ); + }); + it('clicking new run button navigates to the new run page', async () => { - tree = shallow(); + tree = shallow(); await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; const newRunFromPipelineVersionBtn = instance.getInitialToolbarState().actions[ @@ -684,9 +786,7 @@ describe('PipelineDetails', () => { expect(historyPushSpy).toHaveBeenCalledTimes(1); expect(historyPushSpy).toHaveBeenLastCalledWith( RoutePage.NEW_RUN + - `?${QUERY_PARAMS.pipelineId}=${testV1Pipeline.id}&${ - QUERY_PARAMS.pipelineVersionId - }=${testV1Pipeline.default_version!.id!}`, + `?${QUERY_PARAMS.pipelineId}=${testV2Pipeline.pipeline_id}&${QUERY_PARAMS.pipelineVersionId}=${PIPELINE_VERSION_ID}`, ); }); @@ -694,7 +794,7 @@ describe('PipelineDetails', () => { 'clicking new run button when viewing half-loaded page navigates to ' + 'the new run page with pipeline ID and version ID', async () => { - tree = shallow(); + tree = shallow(); // Intentionally don't wait until all network requests finish. const instance = tree.instance() as PipelineDetails; const newRunFromPipelineVersionBtn = instance.getInitialToolbarState().actions[ @@ -704,16 +804,13 @@ describe('PipelineDetails', () => { expect(historyPushSpy).toHaveBeenCalledTimes(1); expect(historyPushSpy).toHaveBeenLastCalledWith( RoutePage.NEW_RUN + - `?${QUERY_PARAMS.pipelineId}=${testV1Pipeline.id}&${ - QUERY_PARAMS.pipelineVersionId - }=${testV1Pipeline.default_version!.id!}`, + `?${QUERY_PARAMS.pipelineId}=${testV2Pipeline.pipeline_id}&${QUERY_PARAMS.pipelineVersionId}=${PIPELINE_VERSION_ID}`, ); }, ); it('clicking new experiment button navigates to new experiment page', async () => { tree = shallow(); - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; const newExperimentBtn = instance.getInitialToolbarState().actions[ButtonKeys.NEW_EXPERIMENT]; @@ -741,8 +838,7 @@ describe('PipelineDetails', () => { ); it('has a delete button and it is enabled for pipeline version deletion', async () => { - tree = shallow(); - await getPipelineVersionTemplateSpy; + tree = shallow(); await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; const deleteBtn = instance.getInitialToolbarState().actions[ButtonKeys.DELETE_RUN]; @@ -752,12 +848,8 @@ describe('PipelineDetails', () => { it('has a delete button, and it is disabled because no version is selected', async () => { let pageProps = generateProps(); - pageProps.match.params = { - [RouteParams.pipelineId]: testV1Pipeline.id, - }; tree = shallow(); - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); const instance = tree.instance() as PipelineDetails; const deleteBtn = instance.getInitialToolbarState().actions[ButtonKeys.DELETE_RUN]; @@ -766,7 +858,7 @@ describe('PipelineDetails', () => { }); it('shows delete confirmation dialog when delete button is clicked', async () => { - tree = shallow(); + tree = shallow(); const deleteBtn = (tree.instance() as PipelineDetails).getInitialToolbarState().actions[ ButtonKeys.DELETE_RUN ]; @@ -780,7 +872,7 @@ describe('PipelineDetails', () => { }); it('does not call delete API for selected pipeline when delete dialog is canceled', async () => { - tree = shallow(); + tree = shallow(); const deleteBtn = (tree.instance() as PipelineDetails).getInitialToolbarState().actions[ ButtonKeys.DELETE_RUN ]; @@ -792,8 +884,7 @@ describe('PipelineDetails', () => { }); it('calls delete API when delete dialog is confirmed', async () => { - tree = shallow(); - await getPipelineVersionTemplateSpy; + tree = shallow(); await TestUtils.flushPromises(); const deleteBtn = (tree.instance() as PipelineDetails).getInitialToolbarState().actions[ ButtonKeys.DELETE_RUN @@ -810,7 +901,7 @@ describe('PipelineDetails', () => { }); it('calls delete API when delete dialog is confirmed and page is half-loaded', async () => { - tree = shallow(); + tree = shallow(); // Intentionally don't wait until all network requests finish. const deleteBtn = (tree.instance() as PipelineDetails).getInitialToolbarState().actions[ ButtonKeys.DELETE_RUN @@ -827,9 +918,8 @@ describe('PipelineDetails', () => { }); it('shows error dialog if deletion fails', async () => { - tree = shallow(); + tree = shallow(); TestUtils.makeErrorResponseOnce(deletePipelineVersionSpy, 'woops'); - await getPipelineVersionTemplateSpy; await TestUtils.flushPromises(); const deleteBtn = (tree.instance() as PipelineDetails).getInitialToolbarState().actions[ ButtonKeys.DELETE_RUN @@ -848,8 +938,7 @@ describe('PipelineDetails', () => { }); it('shows success snackbar if deletion succeeds', async () => { - tree = shallow(); - await getPipelineVersionTemplateSpy; + tree = shallow(); await TestUtils.flushPromises(); const deleteBtn = (tree.instance() as PipelineDetails).getInitialToolbarState().actions[ ButtonKeys.DELETE_RUN diff --git a/frontend/src/pages/PipelineDetails.tsx b/frontend/src/pages/PipelineDetails.tsx index 8109b0fe77..107e0d68f4 100644 --- a/frontend/src/pages/PipelineDetails.tsx +++ b/frontend/src/pages/PipelineDetails.tsx @@ -32,7 +32,7 @@ import * as WorkflowUtils from 'src/lib/v2/WorkflowUtils'; import { convertYamlToV2PipelineSpec } from 'src/lib/v2/WorkflowUtils'; import { classes } from 'typestyle'; import { Workflow } from 'src/third_party/mlmd/argo_template'; -import { ApiGetTemplateResponse, ApiPipeline, ApiPipelineVersion } from 'src/apis/pipeline'; +import { ApiPipeline, ApiPipelineVersion } from 'src/apis/pipeline'; import { V2beta1ListPipelineVersionsResponse, V2beta1Pipeline, @@ -131,10 +131,18 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { buttons .newRunFromPipelineVersion( () => { - return pipelineIdFromParams ? pipelineIdFromParams : ''; + return this.state.v2Pipeline + ? this.state.v2Pipeline.pipeline_id + : pipelineIdFromParams + ? pipelineIdFromParams + : ''; }, () => { - return pipelineVersionIdFromParams ? pipelineVersionIdFromParams : ''; + return this.state.v2SelectedVersion + ? this.state.v2SelectedVersion.pipeline_version_id + : pipelineVersionIdFromParams + ? pipelineVersionIdFromParams + : ''; }, ) .newPipelineVersion('Upload version', () => @@ -200,7 +208,7 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { pipeline={v2Pipeline} selectedVersion={v2SelectedVersion} versions={v2Versions} - handleVersionSelected={this.handleVersionSelectedV2.bind(this)} + handleVersionSelected={this.handleVersionSelected.bind(this)} /> )} {!this.state.graphIsLoading && !showV2Pipeline && ( @@ -273,20 +281,20 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { // We don't have default version in v2 pipeline proto, choose the latest version instead. private async getSelectedVersion(pipelineId: string, versionId?: string) { + let selectedVersion: V2beta1PipelineVersion; // Get specific version if version id is provided if (versionId) { try { - return await Apis.pipelineServiceApiV2.getPipelineVersion(pipelineId, versionId); + selectedVersion = await Apis.pipelineServiceApiV2.getPipelineVersion(pipelineId, versionId); } catch (err) { this.setStateSafe({ graphIsLoading: false }); await this.showPageError('Cannot retrieve pipeline version.', err); logger.error('Cannot retrieve pipeline version.', err); - return; + return undefined; } } else { // Get the latest version if no version id let listVersionsResponse: V2beta1ListPipelineVersionsResponse; - let latesetVersion: V2beta1PipelineVersion; try { listVersionsResponse = await Apis.pipelineServiceApiV2.listPipelineVersions( pipelineId, @@ -295,22 +303,22 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { 'created_at desc', ); - if (listVersionsResponse.pipeline_versions) { - latesetVersion = listVersionsResponse.pipeline_versions[0]; - // Append version id to URL for create run (new run switcher call getPipelineVersion) - this.props.history.replace({ - pathname: `/pipelines/details/${pipelineId}/version/${latesetVersion.pipeline_version_id}`, - }); - return latesetVersion; + if ( + listVersionsResponse.pipeline_versions && + listVersionsResponse.pipeline_versions.length > 0 + ) { + selectedVersion = listVersionsResponse.pipeline_versions[0]; + } else { + return undefined; } - return undefined; } catch (err) { this.setStateSafe({ graphIsLoading: false }); await this.showPageError('Cannot retrieve pipeline version list.', err); logger.error('Cannot retrieve pipeline version list.', err); - return; + return undefined; } } + return selectedVersion; } public async load(): Promise { @@ -523,10 +531,7 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { logger.error('Cannot retrieve pipeline versions.', err); return; } - templateString = await this._getTemplateString( - pipelineId, - v2SelectedVersion ? v2SelectedVersion.pipeline_version_id! : v1SelectedVersion?.id!, - ); + templateString = await this._getTemplateString(v2SelectedVersion); } breadcrumbs = [{ displayName: 'Pipelines', href: RoutePage.PIPELINES }]; @@ -572,64 +577,18 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { } public async handleVersionSelected(versionId: string): Promise { - if (this.state.v1Pipeline) { - const selectedVersionV1 = (this.state.v1Versions || []).find(v => v.id === versionId); - const selectedVersionV2 = (this.state.v2Versions || []).find( - v => v.pipeline_version_id === versionId, - ); - const pageTitle = this.state.v1Pipeline.name?.concat(' (', selectedVersionV1?.name!, ')'); - - const selectedVersionPipelineTemplate = await this._getTemplateString( - this.state.v1Pipeline.id!, - versionId, - ); - this.props.history.replace({ - pathname: `/pipelines/details/${this.state.v1Pipeline.id}/version/${versionId}`, - }); - this.props.updateToolbar(this.getInitialToolbarState()); - this.props.updateToolbar({ pageTitle }); - - const [graph, reducedGraph, graphV2] = await this._createGraph( - selectedVersionPipelineTemplate, - ); - if (isFeatureEnabled(FeatureKey.V2_ALPHA) && graphV2.length > 0) { - this.setStateSafe({ - graph: undefined, - reducedGraph: undefined, - graphV2, - graphIsLoading: false, - v2SelectedVersion: selectedVersionV2, - templateString: selectedVersionPipelineTemplate, - }); - } else { - this.setStateSafe({ - graph, - reducedGraph, - graphV2: undefined, - graphIsLoading: false, - v1SelectedVersion: selectedVersionV1, - templateString: selectedVersionPipelineTemplate, - }); - } - } - } - - public async handleVersionSelectedV2(versionId: string): Promise { if (this.state.v2Pipeline) { - const selectedVersionV1 = (this.state.v1Versions || []).find(v => v.id === versionId); - const selectedVersionV2 = (this.state.v2Versions || []).find( + const v1SelectedVersion = (this.state.v1Versions || []).find(v => v.id === versionId); + const v2SelectedVersion = (this.state.v2Versions || []).find( v => v.pipeline_version_id === versionId, ); const pageTitle = this.state.v2Pipeline.display_name?.concat( ' (', - selectedVersionV2?.display_name!, + v2SelectedVersion?.display_name!, ')', ); - const selectedVersionPipelineTemplate = await this._getTemplateString( - this.state.v2Pipeline.pipeline_id!, - versionId, - ); + const selectedVersionPipelineTemplate = await this._getTemplateString(v2SelectedVersion); this.props.history.replace({ pathname: `/pipelines/details/${this.state.v2Pipeline.pipeline_id}/version/${versionId}`, }); @@ -645,7 +604,7 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { reducedGraph: undefined, graphV2, graphIsLoading: false, - v2SelectedVersion: selectedVersionV2, + v2SelectedVersion, templateString: selectedVersionPipelineTemplate, }); } else { @@ -654,43 +613,20 @@ class PipelineDetails extends Page<{}, PipelineDetailsState> { reducedGraph, graphV2: undefined, graphIsLoading: false, - v1SelectedVersion: selectedVersionV1, + v1SelectedVersion, templateString: selectedVersionPipelineTemplate, }); } } } - private async _getTemplateString(pipelineId: string, versionId: string): Promise { - try { - // Get template string from pipeline_spec in pipeline version (v2 API) - let pipelineVersion; - let pipelineSpecInVersion; - if (pipelineId && versionId) { - pipelineVersion = await Apis.pipelineServiceApiV2.getPipelineVersion(pipelineId, versionId); - pipelineSpecInVersion = pipelineVersion.pipeline_spec; - } - const templateStrFromSpec = pipelineSpecInVersion - ? JsYaml.safeDump(pipelineSpecInVersion) - : ''; - - // Get template string from template or pipeline version template (v1 API) - let templateResponse: ApiGetTemplateResponse; - if (versionId) { - templateResponse = await Apis.pipelineServiceApi.getPipelineVersionTemplate(versionId); - } else { - templateResponse = await Apis.pipelineServiceApi.getTemplate(pipelineId); - } - - return WorkflowUtils.isTemplateV2(templateStrFromSpec) - ? templateStrFromSpec - : templateResponse.template || ''; - } catch (err) { - this.setStateSafe({ graphIsLoading: false }); - await this.showPageError('Cannot retrieve pipeline template.', err); - logger.error('Cannot retrieve pipeline details.', err); + private async _getTemplateString(pipelineVersion?: V2beta1PipelineVersion): Promise { + if (pipelineVersion?.pipeline_spec) { + return JsYaml.safeDump(pipelineVersion.pipeline_spec); + } else { + logger.error('No template string is found'); + return ''; } - return ''; } private async _createGraph( diff --git a/frontend/src/pages/PipelineDetailsTest.test.tsx b/frontend/src/pages/PipelineDetailsTest.test.tsx index 1d64c53241..cd5602a13f 100644 --- a/frontend/src/pages/PipelineDetailsTest.test.tsx +++ b/frontend/src/pages/PipelineDetailsTest.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { render, screen } from '@testing-library/react'; +import { render, screen, waitFor } from '@testing-library/react'; import { graphlib } from 'dagre'; import * as JsYaml from 'js-yaml'; import React from 'react'; @@ -190,12 +190,6 @@ spec: Apis.pipelineServiceApi.listPipelineVersions = jest .fn() .mockResolvedValue({ versions: [testV1PipelineVersion] }); - Apis.pipelineServiceApi.getTemplate = jest - .fn() - .mockResolvedValue({ template: 'test template' }); - Apis.pipelineServiceApi.getPipelineVersionTemplate = jest - .fn() - .mockResolvedValue({ template: 'test template' }); Apis.runServiceApi.getRun = jest.fn().mockResolvedValue(testV1Run); Apis.pipelineServiceApiV2.getPipeline = jest.fn().mockResolvedValue(testV2Pipeline); @@ -221,9 +215,15 @@ spec: jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { return false; }); - Apis.pipelineServiceApi.getPipelineVersionTemplate = jest - .fn() - .mockResolvedValue({ template: 'bad graph' }); + Apis.pipelineServiceApiV2.getPipelineVersion = jest.fn().mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + pipeline_spec: { + apiVersion: 'bad apiversion', + kind: 'bad kind', + }, + }); const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph'); TestUtils.makeErrorResponse(createGraphSpy, 'bad graph'); @@ -247,19 +247,22 @@ spec: jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { return false; }); - Apis.pipelineServiceApi.getPipelineVersionTemplate = jest.fn().mockResolvedValue({ - template: ` - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: entry-point-test- - `, + Apis.pipelineServiceApiV2.getPipelineVersion = jest.fn().mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + pipeline_spec: { + apiVersion: 'argoproj.io/v1alpha1', + kind: 'Workflow', + }, }); const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph'); TestUtils.makeErrorResponse(createGraphSpy, 'bad graph'); - render(); - await TestUtils.flushPromises(); + + await waitFor(() => { + expect(createGraphSpy).toHaveBeenCalled(); + }); screen.getByTestId('pipeline-detail-v1'); expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error @@ -280,16 +283,21 @@ spec: } return false; }); - const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph'); - TestUtils.makeErrorResponse(createGraphSpy, 'bad graph'); Apis.pipelineServiceApiV2.getPipelineVersion = jest.fn().mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', pipeline_spec: JsYaml.safeLoad( 'spec:\n arguments:\n parameters:\n - name: output\n', ), }); - + const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph'); + TestUtils.makeErrorResponse(createGraphSpy, 'bad graph'); render(); - await TestUtils.flushPromises(); + + await waitFor(() => { + expect(createGraphSpy).toHaveBeenCalledTimes(0); + }); screen.getByTestId('pipeline-detail-v1'); expect(updateBannerSpy).toHaveBeenCalledTimes(2); // Once to clear banner, once to show error @@ -313,13 +321,15 @@ spec: const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph'); createGraphSpy.mockImplementation(() => new graphlib.Graph()); - Apis.pipelineServiceApi.getTemplate = jest - .fn() - .mockResolvedValue({ template: v1PipelineSpecTemplate }); - Apis.pipelineServiceApi.getPipelineVersionTemplate = jest - .fn() - .mockResolvedValue({ template: v1PipelineSpecTemplate }); - Apis.pipelineServiceApiV2.getPipelineVersion = jest.fn().mockResolvedValue({}); + Apis.pipelineServiceApiV2.getPipelineVersion = jest.fn().mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + pipeline_spec: { + apiVersion: 'argoproj.io/v1alpha1', + kind: 'Workflow', + }, + }); render(); await TestUtils.flushPromises(); @@ -333,9 +343,15 @@ spec: jest.spyOn(features, 'isFeatureEnabled').mockImplementation(featureKey => { return false; }); - Apis.pipelineServiceApi.getPipelineVersionTemplate = jest - .fn() - .mockResolvedValue({ template: v1PipelineSpecTemplate }); + Apis.pipelineServiceApiV2.getPipelineVersion = jest.fn().mockResolvedValue({ + display_name: 'test-pipeline-version', + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + pipeline_spec: { + apiVersion: 'argoproj.io/v1alpha1', + kind: 'Workflow', + }, + }); const createGraphSpy = jest.spyOn(StaticGraphParser, 'createGraph'); createGraphSpy.mockImplementation(() => new graphlib.Graph()); diff --git a/frontend/src/pages/RecurringRunDetailsRouter.tsx b/frontend/src/pages/RecurringRunDetailsRouter.tsx index c9a276056d..31919e3fa7 100644 --- a/frontend/src/pages/RecurringRunDetailsRouter.tsx +++ b/frontend/src/pages/RecurringRunDetailsRouter.tsx @@ -24,6 +24,8 @@ import * as WorkflowUtils from 'src/lib/v2/WorkflowUtils'; import { PageProps } from './Page'; import RecurringRunDetails from './RecurringRunDetails'; import RecurringRunDetailsV2 from './RecurringRunDetailsV2'; +import { RecurringRunDetailsV2FC } from 'src/pages/functional_components/RecurringRunDetailsV2FC'; +import { FeatureKey, isFeatureEnabled } from 'src/features'; // This is a router to determine whether to show V1 or V2 recurring run details page. export default function RecurringRunDetailsRouter(props: PageProps) { @@ -76,7 +78,11 @@ export default function RecurringRunDetailsRouter(props: PageProps) { if (getRecurringRunSuccess && v2RecurringRun && templateString) { const isV2Pipeline = WorkflowUtils.isPipelineSpec(templateString); if (isV2Pipeline) { - return ; + return isFeatureEnabled(FeatureKey.FUNCTIONAL_COMPONENT) ? ( + + ) : ( + + ); } } diff --git a/frontend/src/pages/RecurringRunDetailsV2.test.tsx b/frontend/src/pages/RecurringRunDetailsV2.test.tsx index e723c639a8..3e9eb3829a 100644 --- a/frontend/src/pages/RecurringRunDetailsV2.test.tsx +++ b/frontend/src/pages/RecurringRunDetailsV2.test.tsx @@ -98,7 +98,9 @@ describe('RecurringRunDetailsV2', () => { }; jest.clearAllMocks(); - jest.spyOn(features, 'isFeatureEnabled').mockReturnValue(true); + jest + .spyOn(features, 'isFeatureEnabled') + .mockImplementation(featureKey => featureKey === features.FeatureKey.V2_ALPHA); getRecurringRunSpy.mockImplementation(() => fullTestV2RecurringRun); getPipelineVersionSpy.mockImplementation(() => testPipelineVersion); diff --git a/frontend/src/pages/ResourceSelector.test.tsx b/frontend/src/pages/ResourceSelector.test.tsx index 68542b684a..41d613927b 100644 --- a/frontend/src/pages/ResourceSelector.test.tsx +++ b/frontend/src/pages/ResourceSelector.test.tsx @@ -154,7 +154,7 @@ describe('ResourceSelector', () => { expect(tree.state('selectedIds')).toEqual([]); (tree.instance() as TestResourceSelector)._selectionChanged([RESOURCES[1].id!]); - expect(selectionChangedCbSpy).toHaveBeenLastCalledWith(RESOURCES[1]); + expect(selectionChangedCbSpy).toHaveBeenLastCalledWith(RESOURCES[1].id!); expect(tree.state('selectedIds')).toEqual([RESOURCES[1].id]); }); @@ -177,20 +177,4 @@ describe('ResourceSelector', () => { RESOURCES[1].id, ]); }); - - it('logs error if selected resource ID is not found in list', async () => { - tree = shallow(); - const consoleSpy = jest.spyOn(console, 'error').mockImplementation(); - await (tree.instance() as TestResourceSelector)._load({}); - - expect(tree.state('selectedIds')).toEqual([]); - - (tree.instance() as TestResourceSelector)._selectionChanged(['id-not-in-list']); - - expect(selectionChangedCbSpy).not.toHaveBeenCalled(); - expect(tree.state('selectedIds')).toEqual([]); - expect(consoleSpy).toHaveBeenLastCalledWith( - 'Somehow no resource was found with ID: id-not-in-list', - ); - }); }); diff --git a/frontend/src/pages/ResourceSelector.tsx b/frontend/src/pages/ResourceSelector.tsx index 1b7987d1e5..be81359ede 100644 --- a/frontend/src/pages/ResourceSelector.tsx +++ b/frontend/src/pages/ResourceSelector.tsx @@ -33,6 +33,7 @@ export interface BaseResource { description?: string; name?: string; error?: string; + namespace?: string; } export interface ResourceSelectorProps extends RouteComponentProps { @@ -41,7 +42,7 @@ export interface ResourceSelectorProps extends RouteComponentProps { emptyMessage: string; filterLabel: string; initialSortColumn: any; - selectionChanged: (resource: BaseResource) => void; + selectionChanged: (selectedId: string) => void; title?: string; toolbarActionMap?: ToolbarActionMap; updateDialog: (dialogProps: DialogProps) => void; @@ -107,13 +108,7 @@ class ResourceSelector extends React.Component r.id === selectedIds[0]); - if (selected) { - this.props.selectionChanged(selected); - } else { - logger.error(`Somehow no resource was found with ID: ${selectedIds[0]}`); - return; - } + this.props.selectionChanged(selectedIds[0]); this.setStateSafe({ selectedIds }); } diff --git a/frontend/src/pages/ResourceSelectorV2.test.tsx b/frontend/src/pages/ResourceSelectorV2.test.tsx deleted file mode 100644 index 7b9dd2d52d..0000000000 --- a/frontend/src/pages/ResourceSelectorV2.test.tsx +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2023 The Kubeflow Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as React from 'react'; -import ResourceSelectorV2, { ResourceSelectorV2Props, BaseResource } from './ResourceSelectorV2'; -import TestUtils from 'src/TestUtils'; -import { ListRequest } from 'src/lib/Apis'; -import { shallow, ReactWrapper, ShallowWrapper } from 'enzyme'; -import { Row } from 'src/components/CustomTable'; - -class TestResourceSelector extends ResourceSelectorV2 { - public async _load(request: ListRequest): Promise { - return super._load(request); - } - - public _selectionChanged(selectedIds: string[]): void { - return super._selectionChanged(selectedIds); - } - - public _resourcesToRow(resources: BaseResource[]): Row[] { - return super._resourcesToRow(resources); - } -} - -describe('ResourceSelector', () => { - let tree: ReactWrapper | ShallowWrapper; - - const updateDialogSpy = jest.fn(); - const selectionChangedCbSpy = jest.fn(); - const listResourceSpy = jest.fn(); - const RESOURCES: BaseResource[] = [ - { - created_at: new Date(2018, 1, 2, 3, 4, 5), - description: 'test-1 description', - id: 'some-id-1', - name: 'test-1 name', - }, - { - created_at: new Date(2018, 10, 9, 8, 7, 6), - description: 'test-2 description', - id: 'some-2-id', - name: 'test-2 name', - }, - ]; - - const selectorColumns = [ - { label: 'Resource name', flex: 1, sortKey: 'name' }, - { label: 'Description', flex: 1.5 }, - { label: 'Uploaded on', flex: 1, sortKey: 'created_at' }, - ]; - - const testEmptyMessage = 'Test - Sorry, no resources.'; - const testTitle = 'A test selector'; - - function generateProps(): ResourceSelectorV2Props { - return { - columns: selectorColumns, - emptyMessage: testEmptyMessage, - filterLabel: 'test filter label', - history: {} as any, - initialSortColumn: 'created_at', - listApi: listResourceSpy as any, - location: '' as any, - match: {} as any, - selectionChanged: selectionChangedCbSpy, - title: testTitle, - updateDialog: updateDialogSpy, - }; - } - - beforeEach(() => { - listResourceSpy.mockReset(); - listResourceSpy.mockImplementation(() => ({ - nextPageToken: 'test-next-page-token', - resources: RESOURCES, - })); - updateDialogSpy.mockReset(); - selectionChangedCbSpy.mockReset(); - }); - - afterEach(async () => { - // unmount() should be called before resetAllMocks() in case any part of the unmount life cycle - // depends on mocks/spies - await tree.unmount(); - }); - - it('displays resource selector', async () => { - tree = shallow(); - await (tree.instance() as TestResourceSelector)._load({}); - - expect(listResourceSpy).toHaveBeenCalledTimes(1); - expect(listResourceSpy).toHaveBeenLastCalledWith(undefined, undefined, undefined, undefined); - expect(tree.state('resources')).toEqual(RESOURCES); - expect(tree).toMatchSnapshot(); - }); - - it('converts resources into a table rows', async () => { - const props = generateProps(); - const resources: BaseResource[] = [ - { - created_at: new Date(2018, 1, 2, 3, 4, 5), - description: 'a description', - id: 'an-id', - name: 'a name', - }, - ]; - listResourceSpy.mockImplementationOnce(() => ({ resources, nextPageToken: '' })); - props.listApi = listResourceSpy as any; - - tree = shallow(); - await (tree.instance() as TestResourceSelector)._load({}); - - expect(tree.state('rows')).toEqual([ - { - id: 'an-id', - otherFields: ['a name', 'a description', '2/2/2018, 3:04:05 AM'], - }, - ]); - }); - - it('shows error dialog if listing fails', async () => { - TestUtils.makeErrorResponseOnce(listResourceSpy, 'woops!'); - jest.spyOn(console, 'error').mockImplementation(); - - tree = shallow(); - await (tree.instance() as TestResourceSelector)._load({}); - - expect(listResourceSpy).toHaveBeenCalledTimes(1); - expect(updateDialogSpy).toHaveBeenLastCalledWith( - expect.objectContaining({ - content: 'List request failed with:\nwoops!', - title: 'Error retrieving resources', - }), - ); - expect(tree.state('resources')).toEqual([]); - }); - - it('calls selection callback when a resource is selected', async () => { - tree = shallow(); - await (tree.instance() as TestResourceSelector)._load({}); - - expect(tree.state('selectedIds')).toEqual([]); - (tree.instance() as TestResourceSelector)._selectionChanged([RESOURCES[1].id!]); - expect(selectionChangedCbSpy).toHaveBeenLastCalledWith(RESOURCES[1].id!); - expect(tree.state('selectedIds')).toEqual([RESOURCES[1].id]); - }); - - it('logs error if more than one resource is selected', async () => { - tree = shallow(); - const consoleSpy = jest.spyOn(console, 'error').mockImplementation(); - await (tree.instance() as TestResourceSelector)._load({}); - - expect(tree.state('selectedIds')).toEqual([]); - - (tree.instance() as TestResourceSelector)._selectionChanged([ - RESOURCES[0].id!, - RESOURCES[1].id!, - ]); - - expect(selectionChangedCbSpy).not.toHaveBeenCalled(); - expect(tree.state('selectedIds')).toEqual([]); - expect(consoleSpy).toHaveBeenLastCalledWith('2 resources were selected somehow', [ - RESOURCES[0].id, - RESOURCES[1].id, - ]); - }); -}); diff --git a/frontend/src/pages/ResourceSelectorV2.tsx b/frontend/src/pages/ResourceSelectorV2.tsx deleted file mode 100644 index 91fe32bae5..0000000000 --- a/frontend/src/pages/ResourceSelectorV2.tsx +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2023 The Kubeflow Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Separate the resource selector between v1 and v2 to avoid breaking current v1 behavior -// TODO(jlyaoyuli): consider to merge 2 selectors together (change updatedSelection() in v1) - -import * as React from 'react'; -import CustomTable, { Column, Row } from 'src/components/CustomTable'; -import Toolbar, { ToolbarActionMap } from 'src/components/Toolbar'; -import { ListRequest } from 'src/lib/Apis'; -import { RouteComponentProps } from 'react-router-dom'; -import { logger, errorToMessage, formatDateString } from 'src/lib/Utils'; -import { DialogProps } from 'src/components/Router'; - -interface BaseResponse { - resources: BaseResource[]; - nextPageToken: string; -} - -export interface BaseResource { - id?: string; - created_at?: Date; - description?: string; - name?: string; - error?: string; - nameSpace?: string; -} - -export interface ResourceSelectorV2Props extends RouteComponentProps { - listApi: (...args: any[]) => Promise; - columns: Column[]; - emptyMessage: string; - filterLabel: string; - initialSortColumn: any; - selectionChanged: (selectedId: string) => void; - title?: string; - toolbarActionMap?: ToolbarActionMap; - updateDialog: (dialogProps: DialogProps) => void; -} - -interface ResourceSelectorV2State { - resources: BaseResource[]; - rows: Row[]; - selectedIds: string[]; - toolbarActionMap: ToolbarActionMap; -} - -class ResourceSelectorV2 extends React.Component { - protected _isMounted = true; - - constructor(props: any) { - super(props); - - this.state = { - resources: [], - rows: [], - selectedIds: [], - toolbarActionMap: (props && props.toolbarActionMap) || {}, - }; - } - - public render(): JSX.Element { - const { rows, selectedIds, toolbarActionMap } = this.state; - const { columns, title, filterLabel, emptyMessage, initialSortColumn } = this.props; - - return ( - - {title && } - - - - ); - } - - public componentWillUnmount(): void { - this._isMounted = false; - } - - protected setStateSafe(newState: Partial, cb?: () => void): void { - if (this._isMounted) { - this.setState(newState as any, cb); - } - } - - protected _selectionChanged(selectedIds: string[]): void { - if (!Array.isArray(selectedIds) || selectedIds.length !== 1) { - logger.error(`${selectedIds.length} resources were selected somehow`, selectedIds); - return; - } - this.props.selectionChanged(selectedIds[0]); - this.setStateSafe({ selectedIds }); - } - - protected async _load(request: ListRequest): Promise { - let nextPageToken = ''; - try { - const response = await this.props.listApi( - request.pageToken, - request.pageSize, - request.sortBy, - request.filter, - ); - - this.setStateSafe({ - resources: response.resources, - rows: this._resourcesToRow(response.resources), - }); - - nextPageToken = response.nextPageToken; - } catch (err) { - const errorMessage = await errorToMessage(err); - this.props.updateDialog({ - buttons: [{ text: 'Dismiss' }], - content: 'List request failed with:\n' + errorMessage, - title: 'Error retrieving resources', - }); - logger.error('Could not get requested list of resources', errorMessage); - } - return nextPageToken; - } - - protected _resourcesToRow(resources: BaseResource[]): Row[] { - return resources.map( - r => - ({ - error: (r as any).error, - id: r.id!, - otherFields: [r.name, r.description, formatDateString(r.created_at)], - } as Row), - ); - } -} - -export default ResourceSelectorV2; diff --git a/frontend/src/pages/__snapshots__/ExperimentDetails.test.tsx.snap b/frontend/src/pages/__snapshots__/ExperimentDetails.test.tsx.snap index 1379caf5b9..d68a007ffb 100644 --- a/frontend/src/pages/__snapshots__/ExperimentDetails.test.tsx.snap +++ b/frontend/src/pages/__snapshots__/ExperimentDetails.test.tsx.snap @@ -19,21 +19,23 @@ exports[`ExperimentDetails fetches this experiment's recurring runs 1`] = `
- Recurring run configs + + Recurring run configs + + + Manage +
1 active
- - Manage - - Recurring run configs + + Recurring run configs + + + Manage +
0 active
- - Manage - - Recurring run configs + + Recurring run configs + + + Manage +
0 active
- - Manage - - Recurring run configs + + Recurring run configs + + + Manage +
0 active
- - Manage - - - - -`; diff --git a/frontend/src/pages/functional_components/NewExperimentFC.test.tsx b/frontend/src/pages/functional_components/NewExperimentFC.test.tsx new file mode 100644 index 0000000000..7ab083a6c5 --- /dev/null +++ b/frontend/src/pages/functional_components/NewExperimentFC.test.tsx @@ -0,0 +1,299 @@ +/* + * Copyright 2023 The Kubeflow Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import * as React from 'react'; +import { CommonTestWrapper } from 'src/TestWrapper'; +import TestUtils from 'src/TestUtils'; +import { NewExperimentFC } from './NewExperimentFC'; +import { Apis } from 'src/lib/Apis'; +import { PageProps } from 'src/pages/Page'; +import * as features from 'src/features'; +import { RoutePage, QUERY_PARAMS } from 'src/components/Router'; + +describe('NewExperiment', () => { + const TEST_EXPERIMENT_ID = 'new-experiment-id'; + const createExperimentSpy = jest.spyOn(Apis.experimentServiceApiV2, 'createExperiment'); + const historyPushSpy = jest.fn(); + const updateDialogSpy = jest.fn(); + const updateSnackbarSpy = jest.fn(); + const updateToolbarSpy = jest.fn(); + + function generateProps(): PageProps { + return { + history: { push: historyPushSpy } as any, + location: { pathname: RoutePage.NEW_EXPERIMENT } as any, + match: '' as any, + toolbarProps: { actions: {}, breadcrumbs: [], pageTitle: TEST_EXPERIMENT_ID }, + updateBanner: () => null, + updateDialog: updateDialogSpy, + updateSnackbar: updateSnackbarSpy, + updateToolbar: updateToolbarSpy, + }; + } + + beforeEach(() => { + jest.clearAllMocks(); + // mock both v2_alpha and functional feature keys are enable. + jest.spyOn(features, 'isFeatureEnabled').mockReturnValue(true); + + createExperimentSpy.mockImplementation(() => ({ + experiment_id: 'new-experiment-id', + display_name: 'new-experiment-name', + })); + }); + + it('does not include any action buttons in the toolbar', () => { + render( + + + , + ); + + expect(updateToolbarSpy).toHaveBeenCalledWith({ + actions: {}, + breadcrumbs: [{ displayName: 'Experiments', href: RoutePage.EXPERIMENTS }], + pageTitle: 'New experiment', + }); + }); + + it("enables the 'Next' button when an experiment name is entered", () => { + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + }); + + it("re-disables the 'Next' button when an experiment name is cleared after having been entered", () => { + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + + // Remove experiment name + fireEvent.change(experimentNameInput, { target: { value: '' } }); + expect(nextButton.closest('button')?.disabled).toEqual(true); + }); + + it('updates the experiment name', () => { + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + expect(experimentNameInput.closest('input')?.value).toBe('new-experiment-name'); + }); + + it('create new experiment', async () => { + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const experimentDescriptionInput = screen.getByLabelText('Description'); + fireEvent.change(experimentDescriptionInput, { + target: { value: 'new-experiment-description' }, + }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + + fireEvent.click(nextButton); + await waitFor(() => { + expect(createExperimentSpy).toHaveBeenCalledWith( + expect.objectContaining({ + description: 'new-experiment-description', + display_name: 'new-experiment-name', + }), + ); + }); + }); + + it('create new experiment with namespace provided', async () => { + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + + fireEvent.click(nextButton); + await waitFor(() => { + expect(createExperimentSpy).toHaveBeenCalledWith( + expect.objectContaining({ + description: '', + display_name: 'new-experiment-name', + namespace: 'test-ns', + }), + ); + }); + }); + + it('navigates to NewRun page upon successful creation', async () => { + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + + fireEvent.click(nextButton); + await waitFor(() => { + expect(createExperimentSpy).toHaveBeenCalledWith( + expect.objectContaining({ + description: '', + display_name: 'new-experiment-name', + }), + ); + }); + expect(historyPushSpy).toHaveBeenCalledWith( + RoutePage.NEW_RUN + `?experimentId=${TEST_EXPERIMENT_ID}` + `&firstRunInExperiment=1`, + ); + }); + + it('includes pipeline ID and version ID in NewRun page query params if present', async () => { + const pipelineId = 'some-pipeline-id'; + const pipelineVersionId = 'version-id'; + const listPipelineVersionsSpy = jest.spyOn(Apis.pipelineServiceApiV2, 'listPipelineVersions'); + listPipelineVersionsSpy.mockImplementation(() => ({ + pipeline_versions: [{ pipeline_version_id: pipelineVersionId }], + })); + + const props = generateProps(); + props.location.search = `?${QUERY_PARAMS.pipelineId}=${pipelineId}`; + + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + + fireEvent.click(nextButton); + await waitFor(() => { + expect(createExperimentSpy).toHaveBeenCalledWith( + expect.objectContaining({ + description: '', + display_name: 'new-experiment-name', + }), + ); + }); + + expect(historyPushSpy).toHaveBeenCalledWith( + RoutePage.NEW_RUN + + `?experimentId=${TEST_EXPERIMENT_ID}` + + `&pipelineId=${pipelineId}` + + `&pipelineVersionId=${pipelineVersionId}` + + `&firstRunInExperiment=1`, + ); + }); + + it('shows snackbar confirmation after experiment is created', async () => { + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + + fireEvent.click(nextButton); + await waitFor(() => { + expect(createExperimentSpy).toHaveBeenCalledWith( + expect.objectContaining({ + description: '', + display_name: 'new-experiment-name', + }), + ); + }); + expect(updateSnackbarSpy).toHaveBeenLastCalledWith({ + autoHideDuration: 10000, + message: 'Successfully created new Experiment: new-experiment-name', + open: true, + }); + }); + + it('shows error dialog when experiment creation fails', async () => { + TestUtils.makeErrorResponseOnce(createExperimentSpy, 'There was something wrong!'); + render( + + + , + ); + + const experimentNameInput = screen.getByLabelText(/Experiment name/); + fireEvent.change(experimentNameInput, { target: { value: 'new-experiment-name' } }); + const nextButton = screen.getByText('Next'); + expect(nextButton.closest('button')?.disabled).toEqual(false); + + fireEvent.click(nextButton); + await waitFor(() => { + expect(createExperimentSpy).toHaveBeenCalled(); + }); + + expect(updateDialogSpy).toHaveBeenCalledWith( + expect.objectContaining({ + content: 'There was something wrong!', + title: 'Experiment creation failed', + }), + ); + }); + + it('navigates to experiment list page upon cancellation', () => { + render( + + + , + ); + + const cancelButton = screen.getByText('Cancel'); + fireEvent.click(cancelButton); + + expect(historyPushSpy).toHaveBeenCalledWith(RoutePage.EXPERIMENTS); + }); +}); diff --git a/frontend/src/pages/functional_components/NewExperimentFC.tsx b/frontend/src/pages/functional_components/NewExperimentFC.tsx new file mode 100644 index 0000000000..4304adfc4c --- /dev/null +++ b/frontend/src/pages/functional_components/NewExperimentFC.tsx @@ -0,0 +1,189 @@ +/* + * Copyright 2023 The Kubeflow Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Button from '@material-ui/core/Button'; +import React, { useEffect, useState } from 'react'; +import { useMutation, useQuery } from 'react-query'; +import { commonCss, fontsize, padding } from 'src/Css'; +import { V2beta1Experiment } from 'src/apisv2beta1/experiment'; +import { V2beta1PipelineVersion } from 'src/apisv2beta1/pipeline'; +import BusyButton from 'src/atoms/BusyButton'; +import Input from 'src/atoms/Input'; +import { QUERY_PARAMS, RoutePage } from 'src/components/Router'; +import { Apis } from 'src/lib/Apis'; +import { URLParser } from 'src/lib/URLParser'; +import { errorToMessage } from 'src/lib/Utils'; +import { getLatestVersion } from 'src/pages/NewRunV2'; +import { PageProps } from 'src/pages/Page'; +import { classes, stylesheet } from 'typestyle'; + +const css = stylesheet({ + errorMessage: { + color: 'red', + }, + // TODO: move to Css.tsx and probably rename. + explanation: { + fontSize: fontsize.small, + }, +}); + +interface ExperimentProps { + namespace?: string; +} + +type NewExperimentFCProps = ExperimentProps & PageProps; + +export function NewExperimentFC(props: NewExperimentFCProps) { + const urlParser = new URLParser(props); + const { namespace, updateDialog, updateSnackbar, updateToolbar } = props; + const [description, setDescription] = useState(''); + const [experimentName, setExperimentName] = useState(''); + const [isbeingCreated, setIsBeingCreated] = useState(false); + const [experimentResponse, setExperimentResponse] = useState(); + const [errMsgFromApi, setErrMsgFromApi] = useState(); + const pipelineId = urlParser.get(QUERY_PARAMS.pipelineId); + + const { data: latestVersion } = useQuery( + ['pipeline_versions', pipelineId], + () => getLatestVersion(pipelineId!), + { enabled: !!pipelineId }, + ); + + useEffect(() => { + updateToolbar({ + actions: {}, + breadcrumbs: [{ displayName: 'Experiments', href: RoutePage.EXPERIMENTS }], + pageTitle: 'New experiment', + }); + // Initialize toolbar only once during the first render. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // Handle the redirection work when createExperiment is succeed + useEffect(() => { + if (experimentResponse) { + const searchString = pipelineId + ? new URLParser(props).build({ + [QUERY_PARAMS.experimentId]: experimentResponse.experiment_id || '', + [QUERY_PARAMS.pipelineId]: pipelineId, + [QUERY_PARAMS.pipelineVersionId]: latestVersion?.pipeline_version_id || '', + [QUERY_PARAMS.firstRunInExperiment]: '1', + }) + : new URLParser(props).build({ + [QUERY_PARAMS.experimentId]: experimentResponse.experiment_id || '', + [QUERY_PARAMS.firstRunInExperiment]: '1', + }); + props.history.push(RoutePage.NEW_RUN + searchString); + + updateSnackbar({ + autoHideDuration: 10000, + message: `Successfully created new Experiment: ${experimentResponse.display_name}`, + open: true, + }); + } + // Only trigger this effect when search string parameters change. + // Do not rerun this effect if updateSnackbar callback has changes to avoid re-rendering. + // Do not rerun this effect if pipelineId has changes to avoid re-rendering. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [experimentResponse, latestVersion]); + + useEffect(() => { + if (errMsgFromApi) { + updateDialog({ + buttons: [{ text: 'Dismiss' }], + onClose: () => setIsBeingCreated(false), + content: errMsgFromApi, + title: 'Experiment creation failed', + }); + } + // Do not rerun this effect if updateDialog callback has changes to avoid re-rendering. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [errMsgFromApi, updateDialog]); + + const newExperimentMutation = useMutation((experiment: V2beta1Experiment) => { + return Apis.experimentServiceApiV2.createExperiment(experiment); + }); + + const createExperiment = () => { + let newExperiment: V2beta1Experiment = { + display_name: experimentName, + description: description, + namespace: namespace, + }; + setIsBeingCreated(true); + + newExperimentMutation.mutate(newExperiment, { + onSuccess: response => { + setExperimentResponse(response); + setErrMsgFromApi(undefined); + }, + onError: async err => { + setErrMsgFromApi(await errorToMessage(err)); + }, + }); + }; + + return ( +
+
+
Experiment details
+
+ Think of an Experiment as a space that contains the history of all pipelines and their + associated runs +
+ + setExperimentName(event.target.value)} + value={experimentName} + autoFocus={true} + variant='outlined' + /> + setDescription(event.target.value)} + required={false} + value={description} + variant='outlined' + /> + +
+ + +
+ {experimentName ? '' : 'Experiment name is required'} +
+
+
+
+ ); +} diff --git a/frontend/src/pages/functional_components/RecurringRunDetailsV2FC.test.tsx b/frontend/src/pages/functional_components/RecurringRunDetailsV2FC.test.tsx new file mode 100644 index 0000000000..800a07e305 --- /dev/null +++ b/frontend/src/pages/functional_components/RecurringRunDetailsV2FC.test.tsx @@ -0,0 +1,370 @@ +/* + * Copyright 2023 The Kubeflow Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { render, screen, waitFor } from '@testing-library/react'; +import * as React from 'react'; +import fs from 'fs'; +import * as JsYaml from 'js-yaml'; +import { CommonTestWrapper } from 'src/TestWrapper'; +import RecurringRunDetailsRouter from 'src/pages/RecurringRunDetailsRouter'; +import TestUtils from 'src/TestUtils'; +import { V2beta1RecurringRun, V2beta1RecurringRunStatus } from 'src/apisv2beta1/recurringrun'; +import { V2beta1PipelineVersion } from 'src/apisv2beta1/pipeline'; +import { Apis } from 'src/lib/Apis'; +import { PageProps } from 'src/pages/Page'; +import { RouteParams, RoutePage } from 'src/components/Router'; +import * as features from 'src/features'; + +const V2_PIPELINESPEC_PATH = 'src/data/test/lightweight_python_functions_v2_pipeline_rev.yaml'; +const v2YamlTemplateString = fs.readFileSync(V2_PIPELINESPEC_PATH, 'utf8'); + +describe('RecurringRunDetailsV2FC', () => { + const updateBannerSpy = jest.fn(); + const updateDialogSpy = jest.fn(); + const updateSnackbarSpy = jest.fn(); + const updateToolbarSpy = jest.fn(); + const historyPushSpy = jest.fn(); + const historyReplaceSpy = jest.fn(); + const getRecurringRunSpy = jest.spyOn(Apis.recurringRunServiceApi, 'getRecurringRun'); + const deleteRecurringRunSpy = jest.spyOn(Apis.recurringRunServiceApi, 'deleteRecurringRun'); + const enableRecurringRunSpy = jest.spyOn(Apis.recurringRunServiceApi, 'enableRecurringRun'); + const disableRecurringRunSpy = jest.spyOn(Apis.recurringRunServiceApi, 'disableRecurringRun'); + const getExperimentSpy = jest.spyOn(Apis.experimentServiceApiV2, 'getExperiment'); + const getPipelineVersionSpy = jest.spyOn(Apis.pipelineServiceApiV2, 'getPipelineVersion'); + + let fullTestV2RecurringRun: V2beta1RecurringRun = {}; + let testPipelineVersion: V2beta1PipelineVersion = {}; + + function generateProps(): PageProps { + return { + history: { push: historyPushSpy, replace: historyReplaceSpy } as any, + location: '' as any, + match: { + params: { [RouteParams.recurringRunId]: fullTestV2RecurringRun.recurring_run_id }, + isExact: true, + path: '', + url: '', + }, + toolbarProps: { actions: {}, breadcrumbs: [], pageTitle: '' }, + updateBanner: updateBannerSpy, + updateDialog: updateDialogSpy, + updateSnackbar: updateSnackbarSpy, + updateToolbar: updateToolbarSpy, + }; + } + + beforeEach(() => { + fullTestV2RecurringRun = { + created_at: new Date(2018, 8, 5, 4, 3, 2), + description: 'test recurring run description', + display_name: 'test recurring run', + max_concurrency: '50', + no_catchup: true, + pipeline_version_reference: { + pipeline_id: 'test-pipeline-id', + pipeline_version_id: 'test-pipeline-version-id', + }, + recurring_run_id: 'test-recurring-run-id', + runtime_config: { parameters: { param1: 'value1' } }, + status: V2beta1RecurringRunStatus.ENABLED, + trigger: { + periodic_schedule: { + end_time: new Date(2018, 10, 9, 8, 7, 6), + interval_second: '3600', + start_time: new Date(2018, 9, 8, 7, 6), + }, + }, + } as V2beta1RecurringRun; + + testPipelineVersion = { + display_name: 'test_pipeline_version', + pipeline_id: 'test_pipeline_id', + pipeline_version_id: 'test_pipeline_version_id', + pipeline_spec: JsYaml.safeLoad(v2YamlTemplateString), + }; + + jest.clearAllMocks(); + // mock both v2_alpha and functional feature keys are enable. + jest.spyOn(features, 'isFeatureEnabled').mockReturnValue(true); + + getRecurringRunSpy.mockImplementation(() => fullTestV2RecurringRun); + getPipelineVersionSpy.mockImplementation(() => testPipelineVersion); + + deleteRecurringRunSpy.mockImplementation(); + enableRecurringRunSpy.mockImplementation(); + disableRecurringRunSpy.mockImplementation(); + getExperimentSpy.mockImplementation(); + }); + + it('renders a recurring run with periodic schedule', async () => { + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalledTimes(2); + expect(getPipelineVersionSpy).toHaveBeenCalled(); + }); + + screen.getByText('Enabled'); + screen.getByText('Yes'); + screen.getByText('Trigger'); + screen.getByText('Every 1 hours'); + screen.getByText('Max. concurrent runs'); + screen.getByText('50'); + screen.getByText('Catchup'); + screen.getByText('false'); + screen.getByText('param1'); + screen.getByText('value1'); + }); + + it('renders a recurring run with cron schedule', async () => { + const cronTestRecurringRun = { + ...fullTestV2RecurringRun, + no_catchup: undefined, // in api requests, it's undefined when false + trigger: { + cron_schedule: { + cron: '* * * 0 0 !', + end_time: new Date(2018, 10, 9, 8, 7, 6), + start_time: new Date(2018, 9, 8, 7, 6), + }, + }, + }; + getRecurringRunSpy.mockImplementation(() => cronTestRecurringRun); + + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + }); + + screen.getByText('Enabled'); + screen.getByText('Yes'); + screen.getByText('Trigger'); + screen.getByText('* * * 0 0 !'); + screen.getByText('Max. concurrent runs'); + screen.getByText('50'); + screen.getByText('Catchup'); + screen.getByText('true'); + }); + + it('loads the recurring run given its id in query params', async () => { + // The run id is in the router match object, defined inside generateProps + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + }); + + expect(getRecurringRunSpy).toHaveBeenLastCalledWith(fullTestV2RecurringRun.recurring_run_id); + expect(getExperimentSpy).not.toHaveBeenCalled(); + }); + + it('shows All runs -> run name when there is no experiment', async () => { + // The run id is in the router match object, defined inside generateProps + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + }); + + expect(updateToolbarSpy).toHaveBeenLastCalledWith( + expect.objectContaining({ + breadcrumbs: [{ displayName: 'All runs', href: RoutePage.RUNS }], + pageTitle: fullTestV2RecurringRun.display_name, + }), + ); + }); + + it('loads the recurring run and its experiment if it has one', async () => { + fullTestV2RecurringRun.experiment_id = 'test-experiment-id'; + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + }); + + expect(getRecurringRunSpy).toHaveBeenLastCalledWith(fullTestV2RecurringRun.recurring_run_id); + expect(getExperimentSpy).toHaveBeenLastCalledWith('test-experiment-id'); + }); + + it('shows Experiments -> Experiment name -> run name when there is an experiment', async () => { + fullTestV2RecurringRun.experiment_id = 'test-experiment-id'; + getExperimentSpy.mockImplementation(id => ({ + experiment_id: id, + display_name: 'test experiment name', + })); + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + expect(getExperimentSpy).toHaveBeenCalled(); + }); + + expect(updateToolbarSpy).toHaveBeenLastCalledWith( + expect.objectContaining({ + breadcrumbs: [ + { displayName: 'Experiments', href: RoutePage.EXPERIMENTS }, + { + displayName: 'test experiment name', + href: RoutePage.EXPERIMENT_DETAILS.replace( + ':' + RouteParams.experimentId, + 'test-experiment-id', + ), + }, + ], + pageTitle: fullTestV2RecurringRun.display_name, + }), + ); + }); + + it('shows error banner if run cannot be fetched', async () => { + TestUtils.makeErrorResponseOnce(getRecurringRunSpy, 'woops!'); + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + }); + + expect(updateBannerSpy).toHaveBeenLastCalledWith( + expect.objectContaining({ + additionalInfo: 'woops!', + message: `Error: failed to retrieve recurring run: ${fullTestV2RecurringRun.recurring_run_id}. Click Details for more information.`, + mode: 'error', + }), + ); + }); + + it('shows warning banner if has experiment but experiment cannot be fetched. still loads run', async () => { + fullTestV2RecurringRun.experiment_id = 'test-experiment-id'; + TestUtils.makeErrorResponseOnce(getExperimentSpy, 'woops!'); + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + }); + + expect(updateBannerSpy).toHaveBeenLastCalledWith( + expect.objectContaining({ + additionalInfo: 'woops!', + message: `Error: failed to retrieve this recurring run's experiment. Click Details for more information.`, + mode: 'warning', + }), + ); + + // "Still loads run" means that the details are still rendered successfully. + screen.getByText('Enabled'); + screen.getByText('Yes'); + screen.getByText('Trigger'); + screen.getByText('Every 1 hours'); + screen.getByText('Max. concurrent runs'); + screen.getByText('50'); + screen.getByText('Catchup'); + screen.getByText('false'); + screen.getByText('param1'); + screen.getByText('value1'); + }); + + it('shows top bar buttons', async () => { + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + expect(updateToolbarSpy).toHaveBeenCalledWith( + expect.objectContaining({ + actions: expect.objectContaining({ + cloneRecurringRun: expect.objectContaining({ title: 'Clone recurring run' }), + refresh: expect.objectContaining({ title: 'Refresh' }), + enableRecurringRun: expect.objectContaining({ title: 'Enable', disabled: true }), + disableRecurringRun: expect.objectContaining({ title: 'Disable', disabled: false }), + deleteRun: expect.objectContaining({ title: 'Delete' }), + }), + }), + ); + }); + }); + + it('enables Enable buttons if the run is disabled', async () => { + fullTestV2RecurringRun.status = V2beta1RecurringRunStatus.DISABLED; + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + expect(updateToolbarSpy).toHaveBeenCalledWith( + expect.objectContaining({ + actions: expect.objectContaining({ + cloneRecurringRun: expect.objectContaining({ title: 'Clone recurring run' }), + refresh: expect.objectContaining({ title: 'Refresh' }), + enableRecurringRun: expect.objectContaining({ title: 'Enable', disabled: false }), + disableRecurringRun: expect.objectContaining({ title: 'Disable', disabled: true }), + deleteRun: expect.objectContaining({ title: 'Delete' }), + }), + }), + ); + }); + }); + + it('shows enables Enable buttons if the run is undefined', async () => { + fullTestV2RecurringRun.status = undefined; + render( + + + , + ); + await waitFor(() => { + expect(getRecurringRunSpy).toHaveBeenCalled(); + expect(updateToolbarSpy).toHaveBeenCalledWith( + expect.objectContaining({ + actions: expect.objectContaining({ + cloneRecurringRun: expect.objectContaining({ title: 'Clone recurring run' }), + refresh: expect.objectContaining({ title: 'Refresh' }), + enableRecurringRun: expect.objectContaining({ title: 'Enable', disabled: false }), + disableRecurringRun: expect.objectContaining({ title: 'Disable', disabled: true }), + deleteRun: expect.objectContaining({ title: 'Delete' }), + }), + }), + ); + }); + }); +}); diff --git a/frontend/src/pages/functional_components/RecurringRunDetailsV2FC.tsx b/frontend/src/pages/functional_components/RecurringRunDetailsV2FC.tsx new file mode 100644 index 0000000000..0972d50fa2 --- /dev/null +++ b/frontend/src/pages/functional_components/RecurringRunDetailsV2FC.tsx @@ -0,0 +1,249 @@ +/* + * Copyright 2023 The Kubeflow Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { useEffect, useState } from 'react'; +import { useQuery } from 'react-query'; +import Buttons, { ButtonKeys } from 'src/lib/Buttons'; +import DetailsTable from 'src/components/DetailsTable'; +import { V2beta1RecurringRun, V2beta1RecurringRunStatus } from 'src/apisv2beta1/recurringrun'; +import { V2beta1Experiment } from 'src/apisv2beta1/experiment'; +import { Apis } from 'src/lib/Apis'; +import { PageProps } from 'src/pages/Page'; +import { RoutePage, RouteParams } from 'src/components/Router'; +import { Breadcrumb, ToolbarProps } from 'src/components/Toolbar'; +import { classes } from 'typestyle'; +import { commonCss, padding } from 'src/Css'; +import { KeyValue } from 'src/lib/StaticGraphParser'; +import { formatDateString, enabledDisplayStringV2, errorToMessage } from 'src/lib/Utils'; +import { triggerDisplayString } from 'src/lib/TriggerUtils'; + +export function RecurringRunDetailsV2FC(props: PageProps) { + const { updateBanner, updateToolbar } = props; + const [refresh, setRefresh] = useState(true); + const [getRecurringRunErrMsg, setGetRecurringRunErrMsg] = useState(''); + const [getExperimentErrMsg, setGetExperimentErrMsg] = useState(''); + + // Related to Api Response + const [experimentName, setExperimentName] = useState(); + const [experimentIdFromApi, setExperimentIdFromApi] = useState(); + const [recurringRunName, setRecurringRunName] = useState(); + const [recurringRunIdFromApi, setRecurringRunIdFromApi] = useState(); + const [recurringRunStatus, setRecurringRunStatus] = useState(); + + const recurringRunId = props.match.params[RouteParams.recurringRunId]; + const Refresh = () => setRefresh(refreshed => !refreshed); + + const { + data: recurringRun, + error: getRecurringRunError, + refetch: refetchRecurringRun, + } = useQuery( + ['recurringRun', recurringRunId], + async () => { + return await Apis.recurringRunServiceApi.getRecurringRun(recurringRunId); + }, + { enabled: !!recurringRunId, staleTime: 0, cacheTime: 0 }, + ); + + const experimentId = recurringRun?.experiment_id!; + const { data: experiment, error: getExperimentError } = useQuery( + ['experiment'], + async () => { + return await Apis.experimentServiceApiV2.getExperiment(experimentId); + }, + { enabled: !!experimentId, staleTime: 0 }, + ); + + useEffect(() => { + if (recurringRun) { + setRecurringRunName(recurringRun.display_name); + setRecurringRunStatus(recurringRun.status); + setRecurringRunIdFromApi(recurringRun.recurring_run_id); + } + }, [recurringRun]); + + useEffect(() => { + if (experiment) { + setExperimentName(experiment.display_name); + setExperimentIdFromApi(experiment.experiment_id); + } + }, [experiment]); + + useEffect(() => { + const toolbarState = getInitialToolbarState(); + + toolbarState.actions[ButtonKeys.ENABLE_RECURRING_RUN].disabled = + recurringRunStatus === V2beta1RecurringRunStatus.ENABLED; + toolbarState.actions[ButtonKeys.DISABLE_RECURRING_RUN].disabled = + recurringRunStatus !== V2beta1RecurringRunStatus.ENABLED; + toolbarState.pageTitle = recurringRunName || recurringRunIdFromApi || 'Unknown recurring run'; + toolbarState.breadcrumbs = getBreadcrumbs(experimentIdFromApi, experimentName); + updateToolbar(toolbarState); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + recurringRunIdFromApi, + recurringRunName, + recurringRunStatus, + experimentIdFromApi, + experimentName, + ]); + + useEffect(() => { + if (getRecurringRunError) { + (async () => { + const errorMessage = await errorToMessage(getRecurringRunError); + setGetRecurringRunErrMsg(errorMessage); + })(); + } + + // getExperimentError is from the getExperiment useQuery which is enabled by the + // experiment ID in recurringRun object. => when getExperimentError changed, + // getRecurringRun useQuery must be successful (getRecurringRunError is null) + if (getExperimentError) { + (async () => { + const errorMessage = await errorToMessage(getExperimentError); + setGetExperimentErrMsg(errorMessage); + })(); + } + }, [getRecurringRunError, getExperimentError]); + + useEffect(() => { + if (getRecurringRunErrMsg) { + updateBanner({ + additionalInfo: getRecurringRunErrMsg ? getRecurringRunErrMsg : undefined, + message: + `Error: failed to retrieve recurring run: ${recurringRunId}.` + + (getRecurringRunErrMsg ? ' Click Details for more information.' : ''), + mode: 'error', + }); + } + + if (getExperimentErrMsg) { + updateBanner({ + additionalInfo: getExperimentErrMsg ? getExperimentErrMsg : undefined, + message: + `Error: failed to retrieve this recurring run's experiment.` + + (getExperimentErrMsg ? ' Click Details for more information.' : ''), + mode: 'warning', + }); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [getRecurringRunErrMsg, getExperimentErrMsg]); + + useEffect(() => { + refetchRecurringRun(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [refresh]); + + const deleteCallback = (_: string[], success: boolean) => { + if (success) { + const breadcrumbs = props.toolbarProps.breadcrumbs; + const previousPage = breadcrumbs.length + ? breadcrumbs[breadcrumbs.length - 1].href + : RoutePage.EXPERIMENTS; + props.history.push(previousPage); + } + }; + + const getInitialToolbarState = (): ToolbarProps => { + const buttons = new Buttons(props, Refresh); + return { + actions: buttons + .cloneRecurringRun(() => (recurringRun ? [recurringRun.recurring_run_id!] : []), true) + .refresh(Refresh) + .enableRecurringRun(() => (recurringRun ? recurringRun.recurring_run_id! : '')) + .disableRecurringRun(() => (recurringRun ? recurringRun.recurring_run_id! : '')) + .delete( + () => (recurringRun ? [recurringRun.recurring_run_id!] : []), + 'recurring run config', + deleteCallback, + true /* useCurrentResource */, + ) + .getToolbarActionMap(), + breadcrumbs: [], + pageTitle: '', + }; + }; + + return ( +
+ {recurringRun && ( +
+
+ + + +
+
+ )} +
+ ); +} + +function getBreadcrumbs(experimentId?: string, experimentName?: string): Breadcrumb[] { + const breadcrumbs: Breadcrumb[] = []; + if (experimentId) { + breadcrumbs.push( + { displayName: 'Experiments', href: RoutePage.EXPERIMENTS }, + { + displayName: experimentName || 'Unknown experiment name', + href: RoutePage.EXPERIMENT_DETAILS.replace(':' + RouteParams.experimentId, experimentId), + }, + ); + } else { + breadcrumbs.push({ displayName: 'All runs', href: RoutePage.RUNS }); + } + + return breadcrumbs; +} + +function getRecurringRunDetails(recurringRun: V2beta1RecurringRun): Array> { + let details: Array> = []; + + details.push(['Description', recurringRun.description!]); + details.push(['Created at', formatDateString(recurringRun.created_at)]); + + return details; +} + +function getRunTriggers(recurringRun: V2beta1RecurringRun): Array> { + let triggers: Array> = []; + + triggers.push(['Enabled', enabledDisplayStringV2(recurringRun.trigger, recurringRun.status!)]); + triggers.push(['Trigger', triggerDisplayString(recurringRun.trigger)]); + triggers.push(['Max. concurrent runs', recurringRun.max_concurrency]); + triggers.push(['Catchup', `${!recurringRun.no_catchup}`]); + triggers.push(['Start time', '']); + + return triggers; +} + +function getRunParameters(recurringRun: V2beta1RecurringRun): Array> { + let parameters: Array> = []; + + parameters = Object.entries(recurringRun.runtime_config?.parameters || []).map(param => [ + param[0] || '', + param[1] || '', + ]); + + return parameters; +} diff --git a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.d.ts b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.d.ts index f109b9df43..e4ec769838 100644 --- a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.d.ts +++ b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.d.ts @@ -1,5 +1,6 @@ import * as jspb from 'google-protobuf' +import * as google_protobuf_any_pb from 'google-protobuf/google/protobuf/any_pb'; import * as google_protobuf_struct_pb from 'google-protobuf/google/protobuf/struct_pb'; import * as google_protobuf_descriptor_pb from 'google-protobuf/google/protobuf/descriptor_pb'; @@ -37,6 +38,14 @@ export class Value extends jspb.Message { hasStructValue(): boolean; clearStructValue(): Value; + getProtoValue(): google_protobuf_any_pb.Any | undefined; + setProtoValue(value?: google_protobuf_any_pb.Any): Value; + hasProtoValue(): boolean; + clearProtoValue(): Value; + + getBoolValue(): boolean; + setBoolValue(value: boolean): Value; + getValueCase(): Value.ValueCase; serializeBinary(): Uint8Array; @@ -53,6 +62,8 @@ export namespace Value { doubleValue: number, stringValue: string, structValue?: google_protobuf_struct_pb.Struct.AsObject, + protoValue?: google_protobuf_any_pb.Any.AsObject, + boolValue: boolean, } export enum ValueCase { @@ -61,6 +72,8 @@ export namespace Value { DOUBLE_VALUE = 2, STRING_VALUE = 3, STRUCT_VALUE = 4, + PROTO_VALUE = 5, + BOOL_VALUE = 6, } } @@ -80,6 +93,9 @@ export class Artifact extends jspb.Message { getUri(): string; setUri(value: string): Artifact; + getExternalId(): string; + setExternalId(value: string): Artifact; + getPropertiesMap(): jspb.Map; clearPropertiesMap(): Artifact; @@ -95,6 +111,11 @@ export class Artifact extends jspb.Message { getLastUpdateTimeSinceEpoch(): number; setLastUpdateTimeSinceEpoch(value: number): Artifact; + getSystemMetadata(): google_protobuf_any_pb.Any | undefined; + setSystemMetadata(value?: google_protobuf_any_pb.Any): Artifact; + hasSystemMetadata(): boolean; + clearSystemMetadata(): Artifact; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): Artifact.AsObject; static toObject(includeInstance: boolean, msg: Artifact): Artifact.AsObject; @@ -110,11 +131,13 @@ export namespace Artifact { typeId: number, type: string, uri: string, + externalId: string, propertiesMap: Array<[string, Value.AsObject]>, customPropertiesMap: Array<[string, Value.AsObject]>, state: Artifact.State, createTimeSinceEpoch: number, lastUpdateTimeSinceEpoch: number, + systemMetadata?: google_protobuf_any_pb.Any.AsObject, } export enum State { @@ -123,6 +146,8 @@ export namespace Artifact { LIVE = 2, MARKED_FOR_DELETION = 3, DELETED = 4, + ABANDONED = 5, + REFERENCE = 6, } } @@ -139,6 +164,9 @@ export class ArtifactType extends jspb.Message { getDescription(): string; setDescription(value: string): ArtifactType; + getExternalId(): string; + setExternalId(value: string): ArtifactType; + getPropertiesMap(): jspb.Map; clearPropertiesMap(): ArtifactType; @@ -159,6 +187,7 @@ export namespace ArtifactType { name: string, version: string, description: string, + externalId: string, propertiesMap: Array<[string, PropertyType]>, baseType: ArtifactType.SystemDefinedBaseType, } @@ -190,6 +219,11 @@ export class Event extends jspb.Message { getMillisecondsSinceEpoch(): number; setMillisecondsSinceEpoch(value: number): Event; + getSystemMetadata(): google_protobuf_any_pb.Any | undefined; + setSystemMetadata(value?: google_protobuf_any_pb.Any): Event; + hasSystemMetadata(): boolean; + clearSystemMetadata(): Event; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): Event.AsObject; static toObject(includeInstance: boolean, msg: Event): Event.AsObject; @@ -205,6 +239,7 @@ export namespace Event { path?: Event.Path.AsObject, type: Event.Type, millisecondsSinceEpoch: number, + systemMetadata?: google_protobuf_any_pb.Any.AsObject, } export class Path extends jspb.Message { @@ -267,6 +302,7 @@ export namespace Event { OUTPUT = 4, INTERNAL_INPUT = 5, INTERNAL_OUTPUT = 6, + PENDING_OUTPUT = 7, } } @@ -283,6 +319,9 @@ export class Execution extends jspb.Message { getType(): string; setType(value: string): Execution; + getExternalId(): string; + setExternalId(value: string): Execution; + getLastKnownState(): Execution.State; setLastKnownState(value: Execution.State): Execution; @@ -298,6 +337,11 @@ export class Execution extends jspb.Message { getLastUpdateTimeSinceEpoch(): number; setLastUpdateTimeSinceEpoch(value: number): Execution; + getSystemMetadata(): google_protobuf_any_pb.Any | undefined; + setSystemMetadata(value?: google_protobuf_any_pb.Any): Execution; + hasSystemMetadata(): boolean; + clearSystemMetadata(): Execution; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): Execution.AsObject; static toObject(includeInstance: boolean, msg: Execution): Execution.AsObject; @@ -312,11 +356,13 @@ export namespace Execution { name: string, typeId: number, type: string, + externalId: string, lastKnownState: Execution.State, propertiesMap: Array<[string, Value.AsObject]>, customPropertiesMap: Array<[string, Value.AsObject]>, createTimeSinceEpoch: number, lastUpdateTimeSinceEpoch: number, + systemMetadata?: google_protobuf_any_pb.Any.AsObject, } export enum State { @@ -343,6 +389,9 @@ export class ExecutionType extends jspb.Message { getDescription(): string; setDescription(value: string): ExecutionType; + getExternalId(): string; + setExternalId(value: string): ExecutionType; + getPropertiesMap(): jspb.Map; clearPropertiesMap(): ExecutionType; @@ -373,6 +422,7 @@ export namespace ExecutionType { name: string, version: string, description: string, + externalId: string, propertiesMap: Array<[string, PropertyType]>, inputType?: ArtifactStructType.AsObject, outputType?: ArtifactStructType.AsObject, @@ -402,6 +452,9 @@ export class ContextType extends jspb.Message { getDescription(): string; setDescription(value: string): ContextType; + getExternalId(): string; + setExternalId(value: string): ContextType; + getPropertiesMap(): jspb.Map; clearPropertiesMap(): ContextType; @@ -422,6 +475,7 @@ export namespace ContextType { name: string, version: string, description: string, + externalId: string, propertiesMap: Array<[string, PropertyType]>, baseType: ContextType.SystemDefinedBaseType, } @@ -444,6 +498,9 @@ export class Context extends jspb.Message { getType(): string; setType(value: string): Context; + getExternalId(): string; + setExternalId(value: string): Context; + getPropertiesMap(): jspb.Map; clearPropertiesMap(): Context; @@ -456,6 +513,11 @@ export class Context extends jspb.Message { getLastUpdateTimeSinceEpoch(): number; setLastUpdateTimeSinceEpoch(value: number): Context; + getSystemMetadata(): google_protobuf_any_pb.Any | undefined; + setSystemMetadata(value?: google_protobuf_any_pb.Any): Context; + hasSystemMetadata(): boolean; + clearSystemMetadata(): Context; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): Context.AsObject; static toObject(includeInstance: boolean, msg: Context): Context.AsObject; @@ -470,10 +532,12 @@ export namespace Context { name: string, typeId: number, type: string, + externalId: string, propertiesMap: Array<[string, Value.AsObject]>, customPropertiesMap: Array<[string, Value.AsObject]>, createTimeSinceEpoch: number, lastUpdateTimeSinceEpoch: number, + systemMetadata?: google_protobuf_any_pb.Any.AsObject, } } @@ -953,6 +1017,93 @@ export namespace SqliteMetadataSourceConfig { } } +export class PostgreSQLDatabaseConfig extends jspb.Message { + getHost(): string; + setHost(value: string): PostgreSQLDatabaseConfig; + + getHostaddr(): string; + setHostaddr(value: string): PostgreSQLDatabaseConfig; + + getPort(): string; + setPort(value: string): PostgreSQLDatabaseConfig; + + getUser(): string; + setUser(value: string): PostgreSQLDatabaseConfig; + + getPassword(): string; + setPassword(value: string): PostgreSQLDatabaseConfig; + + getPassfile(): string; + setPassfile(value: string): PostgreSQLDatabaseConfig; + + getDbname(): string; + setDbname(value: string): PostgreSQLDatabaseConfig; + + getSkipDbCreation(): boolean; + setSkipDbCreation(value: boolean): PostgreSQLDatabaseConfig; + + getSsloption(): PostgreSQLDatabaseConfig.SSLOptions | undefined; + setSsloption(value?: PostgreSQLDatabaseConfig.SSLOptions): PostgreSQLDatabaseConfig; + hasSsloption(): boolean; + clearSsloption(): PostgreSQLDatabaseConfig; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): PostgreSQLDatabaseConfig.AsObject; + static toObject(includeInstance: boolean, msg: PostgreSQLDatabaseConfig): PostgreSQLDatabaseConfig.AsObject; + static serializeBinaryToWriter(message: PostgreSQLDatabaseConfig, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): PostgreSQLDatabaseConfig; + static deserializeBinaryFromReader(message: PostgreSQLDatabaseConfig, reader: jspb.BinaryReader): PostgreSQLDatabaseConfig; +} + +export namespace PostgreSQLDatabaseConfig { + export type AsObject = { + host: string, + hostaddr: string, + port: string, + user: string, + password: string, + passfile: string, + dbname: string, + skipDbCreation: boolean, + ssloption?: PostgreSQLDatabaseConfig.SSLOptions.AsObject, + } + + export class SSLOptions extends jspb.Message { + getSslmode(): string; + setSslmode(value: string): SSLOptions; + + getSslcert(): string; + setSslcert(value: string): SSLOptions; + + getSslkey(): string; + setSslkey(value: string): SSLOptions; + + getSslpassword(): string; + setSslpassword(value: string): SSLOptions; + + getSslrootcert(): string; + setSslrootcert(value: string): SSLOptions; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): SSLOptions.AsObject; + static toObject(includeInstance: boolean, msg: SSLOptions): SSLOptions.AsObject; + static serializeBinaryToWriter(message: SSLOptions, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): SSLOptions; + static deserializeBinaryFromReader(message: SSLOptions, reader: jspb.BinaryReader): SSLOptions; + } + + export namespace SSLOptions { + export type AsObject = { + sslmode: string, + sslcert: string, + sslkey: string, + sslpassword: string, + sslrootcert: string, + } + } + +} + export class MigrationOptions extends jspb.Message { getEnableUpgradeMigration(): boolean; setEnableUpgradeMigration(value: boolean): MigrationOptions; @@ -1009,6 +1160,11 @@ export class ConnectionConfig extends jspb.Message { hasSqlite(): boolean; clearSqlite(): ConnectionConfig; + getPostgresql(): PostgreSQLDatabaseConfig | undefined; + setPostgresql(value?: PostgreSQLDatabaseConfig): ConnectionConfig; + hasPostgresql(): boolean; + clearPostgresql(): ConnectionConfig; + getRetryOptions(): RetryOptions | undefined; setRetryOptions(value?: RetryOptions): ConnectionConfig; hasRetryOptions(): boolean; @@ -1029,6 +1185,7 @@ export namespace ConnectionConfig { fakeDatabase?: FakeDatabaseConfig.AsObject, mysql?: MySQLDatabaseConfig.AsObject, sqlite?: SqliteMetadataSourceConfig.AsObject, + postgresql?: PostgreSQLDatabaseConfig.AsObject, retryOptions?: RetryOptions.AsObject, } @@ -1037,6 +1194,7 @@ export namespace ConnectionConfig { FAKE_DATABASE = 1, MYSQL = 2, SQLITE = 3, + POSTGRESQL = 5, } } @@ -1287,6 +1445,9 @@ export namespace ListOperationNextPageToken { } export class TransactionOptions extends jspb.Message { + getTag(): string; + setTag(value: string): TransactionOptions; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): TransactionOptions.AsObject; static toObject(includeInstance: boolean, msg: TransactionOptions): TransactionOptions.AsObject; @@ -1297,6 +1458,7 @@ export class TransactionOptions extends jspb.Message { export namespace TransactionOptions { export type AsObject = { + tag: string, } } @@ -1311,6 +1473,9 @@ export class LineageGraphQueryOptions extends jspb.Message { hasStopConditions(): boolean; clearStopConditions(): LineageGraphQueryOptions; + getMaxNodeSize(): number; + setMaxNodeSize(value: number): LineageGraphQueryOptions; + getQueryNodesCase(): LineageGraphQueryOptions.QueryNodesCase; serializeBinary(): Uint8Array; @@ -1325,6 +1490,7 @@ export namespace LineageGraphQueryOptions { export type AsObject = { artifactsOptions?: ListOperationOptions.AsObject, stopConditions?: LineageGraphQueryOptions.BoundaryConstraint.AsObject, + maxNodeSize: number, } export class BoundaryConstraint extends jspb.Message { @@ -1360,10 +1526,80 @@ export namespace LineageGraphQueryOptions { } } +export class LineageSubgraphQueryOptions extends jspb.Message { + getStartingArtifacts(): LineageSubgraphQueryOptions.StartingNodes | undefined; + setStartingArtifacts(value?: LineageSubgraphQueryOptions.StartingNodes): LineageSubgraphQueryOptions; + hasStartingArtifacts(): boolean; + clearStartingArtifacts(): LineageSubgraphQueryOptions; + + getStartingExecutions(): LineageSubgraphQueryOptions.StartingNodes | undefined; + setStartingExecutions(value?: LineageSubgraphQueryOptions.StartingNodes): LineageSubgraphQueryOptions; + hasStartingExecutions(): boolean; + clearStartingExecutions(): LineageSubgraphQueryOptions; + + getMaxNumHops(): number; + setMaxNumHops(value: number): LineageSubgraphQueryOptions; + + getDirection(): LineageSubgraphQueryOptions.Direction; + setDirection(value: LineageSubgraphQueryOptions.Direction): LineageSubgraphQueryOptions; + + getStartingNodesCase(): LineageSubgraphQueryOptions.StartingNodesCase; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): LineageSubgraphQueryOptions.AsObject; + static toObject(includeInstance: boolean, msg: LineageSubgraphQueryOptions): LineageSubgraphQueryOptions.AsObject; + static serializeBinaryToWriter(message: LineageSubgraphQueryOptions, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): LineageSubgraphQueryOptions; + static deserializeBinaryFromReader(message: LineageSubgraphQueryOptions, reader: jspb.BinaryReader): LineageSubgraphQueryOptions; +} + +export namespace LineageSubgraphQueryOptions { + export type AsObject = { + startingArtifacts?: LineageSubgraphQueryOptions.StartingNodes.AsObject, + startingExecutions?: LineageSubgraphQueryOptions.StartingNodes.AsObject, + maxNumHops: number, + direction: LineageSubgraphQueryOptions.Direction, + } + + export class StartingNodes extends jspb.Message { + getFilterQuery(): string; + setFilterQuery(value: string): StartingNodes; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): StartingNodes.AsObject; + static toObject(includeInstance: boolean, msg: StartingNodes): StartingNodes.AsObject; + static serializeBinaryToWriter(message: StartingNodes, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): StartingNodes; + static deserializeBinaryFromReader(message: StartingNodes, reader: jspb.BinaryReader): StartingNodes; + } + + export namespace StartingNodes { + export type AsObject = { + filterQuery: string, + } + } + + + export enum Direction { + DIRECTION_UNSPECIFIED = 0, + UPSTREAM = 1, + DOWNSTREAM = 2, + BIDIRECTIONAL = 3, + } + + export enum StartingNodesCase { + STARTING_NODES_NOT_SET = 0, + STARTING_ARTIFACTS = 1, + STARTING_EXECUTIONS = 2, + } +} + export enum PropertyType { UNKNOWN = 0, INT = 1, DOUBLE = 2, STRING = 3, STRUCT = 4, + PROTO = 5, + BOOLEAN = 6, } diff --git a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.js b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.js index 996c817cd1..c3801981dc 100644 --- a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.js +++ b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_pb.js @@ -15,6 +15,8 @@ var jspb = require('google-protobuf'); var goog = jspb; var global = Function('return this')(); +var google_protobuf_any_pb = require('google-protobuf/google/protobuf/any_pb.js'); +goog.object.extend(proto, google_protobuf_any_pb); var google_protobuf_struct_pb = require('google-protobuf/google/protobuf/struct_pb.js'); goog.object.extend(proto, google_protobuf_struct_pb); var google_protobuf_descriptor_pb = require('google-protobuf/google/protobuf/descriptor_pb.js'); @@ -50,6 +52,10 @@ goog.exportSymbol('proto.ml_metadata.LineageGraph', null, global); goog.exportSymbol('proto.ml_metadata.LineageGraphQueryOptions', null, global); goog.exportSymbol('proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint', null, global); goog.exportSymbol('proto.ml_metadata.LineageGraphQueryOptions.QueryNodesCase', null, global); +goog.exportSymbol('proto.ml_metadata.LineageSubgraphQueryOptions', null, global); +goog.exportSymbol('proto.ml_metadata.LineageSubgraphQueryOptions.Direction', null, global); +goog.exportSymbol('proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes', null, global); +goog.exportSymbol('proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodesCase', null, global); goog.exportSymbol('proto.ml_metadata.ListArtifactStructType', null, global); goog.exportSymbol('proto.ml_metadata.ListOperationNextPageToken', null, global); goog.exportSymbol('proto.ml_metadata.ListOperationOptions', null, global); @@ -64,6 +70,8 @@ goog.exportSymbol('proto.ml_metadata.MySQLDatabaseConfig', null, global); goog.exportSymbol('proto.ml_metadata.MySQLDatabaseConfig.SSLOptions', null, global); goog.exportSymbol('proto.ml_metadata.NoneArtifactStructType', null, global); goog.exportSymbol('proto.ml_metadata.ParentContext', null, global); +goog.exportSymbol('proto.ml_metadata.PostgreSQLDatabaseConfig', null, global); +goog.exportSymbol('proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions', null, global); goog.exportSymbol('proto.ml_metadata.PropertyType', null, global); goog.exportSymbol('proto.ml_metadata.RetryOptions', null, global); goog.exportSymbol('proto.ml_metadata.SqliteMetadataSourceConfig', null, global); @@ -642,6 +650,48 @@ if (goog.DEBUG && !COMPILED) { */ proto.ml_metadata.SqliteMetadataSourceConfig.displayName = 'proto.ml_metadata.SqliteMetadataSourceConfig'; } +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.PostgreSQLDatabaseConfig = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.PostgreSQLDatabaseConfig, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.PostgreSQLDatabaseConfig.displayName = 'proto.ml_metadata.PostgreSQLDatabaseConfig'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.displayName = 'proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions'; +} /** * Generated by JsPbCodeGenerator. * @param {Array=} opt_data Optional initial data array, typically from a @@ -884,7 +934,7 @@ if (goog.DEBUG && !COMPILED) { * @constructor */ proto.ml_metadata.TransactionOptions = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, 1, null, null); + jspb.Message.initialize(this, opt_data, 0, 2, null, null); }; goog.inherits(proto.ml_metadata.TransactionOptions, jspb.Message); if (goog.DEBUG && !COMPILED) { @@ -966,6 +1016,48 @@ if (goog.DEBUG && !COMPILED) { */ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.displayName = 'proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint'; } +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.LineageSubgraphQueryOptions = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, proto.ml_metadata.LineageSubgraphQueryOptions.oneofGroups_); +}; +goog.inherits(proto.ml_metadata.LineageSubgraphQueryOptions, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.LineageSubgraphQueryOptions.displayName = 'proto.ml_metadata.LineageSubgraphQueryOptions'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.displayName = 'proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes'; +} @@ -1123,7 +1215,7 @@ proto.ml_metadata.SystemTypeExtension.prototype.hasTypeName = function() { * @private {!Array>} * @const */ -proto.ml_metadata.Value.oneofGroups_ = [[1,2,3,4]]; +proto.ml_metadata.Value.oneofGroups_ = [[1,2,3,4,5,6]]; /** * @enum {number} @@ -1133,7 +1225,9 @@ proto.ml_metadata.Value.ValueCase = { INT_VALUE: 1, DOUBLE_VALUE: 2, STRING_VALUE: 3, - STRUCT_VALUE: 4 + STRUCT_VALUE: 4, + PROTO_VALUE: 5, + BOOL_VALUE: 6 }; /** @@ -1177,7 +1271,9 @@ proto.ml_metadata.Value.toObject = function(includeInstance, msg) { intValue: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, doubleValue: (f = jspb.Message.getOptionalFloatingPointField(msg, 2)) == null ? undefined : f, stringValue: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, - structValue: (f = msg.getStructValue()) && google_protobuf_struct_pb.Struct.toObject(includeInstance, f) + structValue: (f = msg.getStructValue()) && google_protobuf_struct_pb.Struct.toObject(includeInstance, f), + protoValue: (f = msg.getProtoValue()) && google_protobuf_any_pb.Any.toObject(includeInstance, f), + boolValue: (f = jspb.Message.getBooleanField(msg, 6)) == null ? undefined : f }; if (includeInstance) { @@ -1231,6 +1327,15 @@ proto.ml_metadata.Value.deserializeBinaryFromReader = function(msg, reader) { reader.readMessage(value,google_protobuf_struct_pb.Struct.deserializeBinaryFromReader); msg.setStructValue(value); break; + case 5: + var value = new google_protobuf_any_pb.Any; + reader.readMessage(value,google_protobuf_any_pb.Any.deserializeBinaryFromReader); + msg.setProtoValue(value); + break; + case 6: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setBoolValue(value); + break; default: reader.skipField(); break; @@ -1289,6 +1394,21 @@ proto.ml_metadata.Value.serializeBinaryToWriter = function(message, writer) { google_protobuf_struct_pb.Struct.serializeBinaryToWriter ); } + f = message.getProtoValue(); + if (f != null) { + writer.writeMessage( + 5, + f, + google_protobuf_any_pb.Any.serializeBinaryToWriter + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 6)); + if (f != null) { + writer.writeBool( + 6, + f + ); + } }; @@ -1437,6 +1557,79 @@ proto.ml_metadata.Value.prototype.hasStructValue = function() { }; +/** + * optional google.protobuf.Any proto_value = 5; + * @return {?proto.google.protobuf.Any} + */ +proto.ml_metadata.Value.prototype.getProtoValue = function() { + return /** @type{?proto.google.protobuf.Any} */ ( + jspb.Message.getWrapperField(this, google_protobuf_any_pb.Any, 5)); +}; + + +/** + * @param {?proto.google.protobuf.Any|undefined} value + * @return {!proto.ml_metadata.Value} returns this +*/ +proto.ml_metadata.Value.prototype.setProtoValue = function(value) { + return jspb.Message.setOneofWrapperField(this, 5, proto.ml_metadata.Value.oneofGroups_[0], value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.Value} returns this + */ +proto.ml_metadata.Value.prototype.clearProtoValue = function() { + return this.setProtoValue(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Value.prototype.hasProtoValue = function() { + return jspb.Message.getField(this, 5) != null; +}; + + +/** + * optional bool bool_value = 6; + * @return {boolean} + */ +proto.ml_metadata.Value.prototype.getBoolValue = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 6, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.Value} returns this + */ +proto.ml_metadata.Value.prototype.setBoolValue = function(value) { + return jspb.Message.setOneofField(this, 6, proto.ml_metadata.Value.oneofGroups_[0], value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.Value} returns this + */ +proto.ml_metadata.Value.prototype.clearBoolValue = function() { + return jspb.Message.setOneofField(this, 6, proto.ml_metadata.Value.oneofGroups_[0], undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Value.prototype.hasBoolValue = function() { + return jspb.Message.getField(this, 6) != null; +}; + + @@ -1474,11 +1667,13 @@ proto.ml_metadata.Artifact.toObject = function(includeInstance, msg) { typeId: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, type: (f = jspb.Message.getField(msg, 8)) == null ? undefined : f, uri: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + externalId: (f = jspb.Message.getField(msg, 11)) == null ? undefined : f, propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.Value.toObject) : [], customPropertiesMap: (f = msg.getCustomPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.Value.toObject) : [], state: (f = jspb.Message.getField(msg, 6)) == null ? undefined : f, createTimeSinceEpoch: (f = jspb.Message.getField(msg, 9)) == null ? undefined : f, - lastUpdateTimeSinceEpoch: (f = jspb.Message.getField(msg, 10)) == null ? undefined : f + lastUpdateTimeSinceEpoch: (f = jspb.Message.getField(msg, 10)) == null ? undefined : f, + systemMetadata: (f = msg.getSystemMetadata()) && google_protobuf_any_pb.Any.toObject(includeInstance, f) }; if (includeInstance) { @@ -1535,6 +1730,10 @@ proto.ml_metadata.Artifact.deserializeBinaryFromReader = function(msg, reader) { var value = /** @type {string} */ (reader.readString()); msg.setUri(value); break; + case 11: + var value = /** @type {string} */ (reader.readString()); + msg.setExternalId(value); + break; case 4: var value = msg.getPropertiesMap(); reader.readMessage(value, function(message, reader) { @@ -1559,6 +1758,11 @@ proto.ml_metadata.Artifact.deserializeBinaryFromReader = function(msg, reader) { var value = /** @type {number} */ (reader.readInt64()); msg.setLastUpdateTimeSinceEpoch(value); break; + case 12: + var value = new google_protobuf_any_pb.Any; + reader.readMessage(value,google_protobuf_any_pb.Any.deserializeBinaryFromReader); + msg.setSystemMetadata(value); + break; default: reader.skipField(); break; @@ -1623,6 +1827,13 @@ proto.ml_metadata.Artifact.serializeBinaryToWriter = function(message, writer) { f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 11)); + if (f != null) { + writer.writeString( + 11, + f + ); + } f = message.getPropertiesMap(true); if (f && f.getLength() > 0) { f.serializeBinary(4, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeMessage, proto.ml_metadata.Value.serializeBinaryToWriter); @@ -1652,6 +1863,14 @@ proto.ml_metadata.Artifact.serializeBinaryToWriter = function(message, writer) { f ); } + f = message.getSystemMetadata(); + if (f != null) { + writer.writeMessage( + 12, + f, + google_protobuf_any_pb.Any.serializeBinaryToWriter + ); + } }; @@ -1663,7 +1882,9 @@ proto.ml_metadata.Artifact.State = { PENDING: 1, LIVE: 2, MARKED_FOR_DELETION: 3, - DELETED: 4 + DELETED: 4, + ABANDONED: 5, + REFERENCE: 6 }; /** @@ -1846,6 +2067,42 @@ proto.ml_metadata.Artifact.prototype.hasUri = function() { }; +/** + * optional string external_id = 11; + * @return {string} + */ +proto.ml_metadata.Artifact.prototype.getExternalId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 11, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.Artifact} returns this + */ +proto.ml_metadata.Artifact.prototype.setExternalId = function(value) { + return jspb.Message.setField(this, 11, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.Artifact} returns this + */ +proto.ml_metadata.Artifact.prototype.clearExternalId = function() { + return jspb.Message.setField(this, 11, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Artifact.prototype.hasExternalId = function() { + return jspb.Message.getField(this, 11) != null; +}; + + /** * map properties = 4; * @param {boolean=} opt_noLazyCreate Do not create the map if @@ -1998,6 +2255,43 @@ proto.ml_metadata.Artifact.prototype.hasLastUpdateTimeSinceEpoch = function() { }; +/** + * optional google.protobuf.Any system_metadata = 12; + * @return {?proto.google.protobuf.Any} + */ +proto.ml_metadata.Artifact.prototype.getSystemMetadata = function() { + return /** @type{?proto.google.protobuf.Any} */ ( + jspb.Message.getWrapperField(this, google_protobuf_any_pb.Any, 12)); +}; + + +/** + * @param {?proto.google.protobuf.Any|undefined} value + * @return {!proto.ml_metadata.Artifact} returns this +*/ +proto.ml_metadata.Artifact.prototype.setSystemMetadata = function(value) { + return jspb.Message.setWrapperField(this, 12, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.Artifact} returns this + */ +proto.ml_metadata.Artifact.prototype.clearSystemMetadata = function() { + return this.setSystemMetadata(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Artifact.prototype.hasSystemMetadata = function() { + return jspb.Message.getField(this, 12) != null; +}; + + @@ -2034,6 +2328,7 @@ proto.ml_metadata.ArtifactType.toObject = function(includeInstance, msg) { name: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, version: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f, description: (f = jspb.Message.getField(msg, 5)) == null ? undefined : f, + externalId: (f = jspb.Message.getField(msg, 7)) == null ? undefined : f, propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, undefined) : [], baseType: (f = jspb.Message.getField(msg, 6)) == null ? undefined : f }; @@ -2088,6 +2383,10 @@ proto.ml_metadata.ArtifactType.deserializeBinaryFromReader = function(msg, reade var value = /** @type {string} */ (reader.readString()); msg.setDescription(value); break; + case 7: + var value = /** @type {string} */ (reader.readString()); + msg.setExternalId(value); + break; case 3: var value = msg.getPropertiesMap(); reader.readMessage(value, function(message, reader) { @@ -2155,6 +2454,13 @@ proto.ml_metadata.ArtifactType.serializeBinaryToWriter = function(message, write f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 7)); + if (f != null) { + writer.writeString( + 7, + f + ); + } f = message.getPropertiesMap(true); if (f && f.getLength() > 0) { f.serializeBinary(3, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeEnum); @@ -2324,6 +2630,42 @@ proto.ml_metadata.ArtifactType.prototype.hasDescription = function() { }; +/** + * optional string external_id = 7; + * @return {string} + */ +proto.ml_metadata.ArtifactType.prototype.getExternalId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 7, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.ArtifactType} returns this + */ +proto.ml_metadata.ArtifactType.prototype.setExternalId = function(value) { + return jspb.Message.setField(this, 7, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ArtifactType} returns this + */ +proto.ml_metadata.ArtifactType.prototype.clearExternalId = function() { + return jspb.Message.setField(this, 7, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ArtifactType.prototype.hasExternalId = function() { + return jspb.Message.getField(this, 7) != null; +}; + + /** * map properties = 3; * @param {boolean=} opt_noLazyCreate Do not create the map if @@ -2418,7 +2760,8 @@ proto.ml_metadata.Event.toObject = function(includeInstance, msg) { executionId: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, path: (f = msg.getPath()) && proto.ml_metadata.Event.Path.toObject(includeInstance, f), type: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f, - millisecondsSinceEpoch: (f = jspb.Message.getField(msg, 5)) == null ? undefined : f + millisecondsSinceEpoch: (f = jspb.Message.getField(msg, 5)) == null ? undefined : f, + systemMetadata: (f = msg.getSystemMetadata()) && google_protobuf_any_pb.Any.toObject(includeInstance, f) }; if (includeInstance) { @@ -2476,6 +2819,11 @@ proto.ml_metadata.Event.deserializeBinaryFromReader = function(msg, reader) { var value = /** @type {number} */ (reader.readInt64()); msg.setMillisecondsSinceEpoch(value); break; + case 6: + var value = new google_protobuf_any_pb.Any; + reader.readMessage(value,google_protobuf_any_pb.Any.deserializeBinaryFromReader); + msg.setSystemMetadata(value); + break; default: reader.skipField(); break; @@ -2541,10 +2889,18 @@ proto.ml_metadata.Event.serializeBinaryToWriter = function(message, writer) { f ); } -}; - - -/** + f = message.getSystemMetadata(); + if (f != null) { + writer.writeMessage( + 6, + f, + google_protobuf_any_pb.Any.serializeBinaryToWriter + ); + } +}; + + +/** * @enum {number} */ proto.ml_metadata.Event.Type = { @@ -2554,7 +2910,8 @@ proto.ml_metadata.Event.Type = { INPUT: 3, OUTPUT: 4, INTERNAL_INPUT: 5, - INTERNAL_OUTPUT: 6 + INTERNAL_OUTPUT: 6, + PENDING_OUTPUT: 7 }; @@ -3120,6 +3477,43 @@ proto.ml_metadata.Event.prototype.hasMillisecondsSinceEpoch = function() { }; +/** + * optional google.protobuf.Any system_metadata = 6; + * @return {?proto.google.protobuf.Any} + */ +proto.ml_metadata.Event.prototype.getSystemMetadata = function() { + return /** @type{?proto.google.protobuf.Any} */ ( + jspb.Message.getWrapperField(this, google_protobuf_any_pb.Any, 6)); +}; + + +/** + * @param {?proto.google.protobuf.Any|undefined} value + * @return {!proto.ml_metadata.Event} returns this +*/ +proto.ml_metadata.Event.prototype.setSystemMetadata = function(value) { + return jspb.Message.setWrapperField(this, 6, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.Event} returns this + */ +proto.ml_metadata.Event.prototype.clearSystemMetadata = function() { + return this.setSystemMetadata(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Event.prototype.hasSystemMetadata = function() { + return jspb.Message.getField(this, 6) != null; +}; + + @@ -3156,11 +3550,13 @@ proto.ml_metadata.Execution.toObject = function(includeInstance, msg) { name: (f = jspb.Message.getField(msg, 6)) == null ? undefined : f, typeId: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, type: (f = jspb.Message.getField(msg, 7)) == null ? undefined : f, + externalId: (f = jspb.Message.getField(msg, 10)) == null ? undefined : f, lastKnownState: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.Value.toObject) : [], customPropertiesMap: (f = msg.getCustomPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.Value.toObject) : [], createTimeSinceEpoch: (f = jspb.Message.getField(msg, 8)) == null ? undefined : f, - lastUpdateTimeSinceEpoch: (f = jspb.Message.getField(msg, 9)) == null ? undefined : f + lastUpdateTimeSinceEpoch: (f = jspb.Message.getField(msg, 9)) == null ? undefined : f, + systemMetadata: (f = msg.getSystemMetadata()) && google_protobuf_any_pb.Any.toObject(includeInstance, f) }; if (includeInstance) { @@ -3213,6 +3609,10 @@ proto.ml_metadata.Execution.deserializeBinaryFromReader = function(msg, reader) var value = /** @type {string} */ (reader.readString()); msg.setType(value); break; + case 10: + var value = /** @type {string} */ (reader.readString()); + msg.setExternalId(value); + break; case 3: var value = /** @type {!proto.ml_metadata.Execution.State} */ (reader.readEnum()); msg.setLastKnownState(value); @@ -3237,6 +3637,11 @@ proto.ml_metadata.Execution.deserializeBinaryFromReader = function(msg, reader) var value = /** @type {number} */ (reader.readInt64()); msg.setLastUpdateTimeSinceEpoch(value); break; + case 11: + var value = new google_protobuf_any_pb.Any; + reader.readMessage(value,google_protobuf_any_pb.Any.deserializeBinaryFromReader); + msg.setSystemMetadata(value); + break; default: reader.skipField(); break; @@ -3294,6 +3699,13 @@ proto.ml_metadata.Execution.serializeBinaryToWriter = function(message, writer) f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 10)); + if (f != null) { + writer.writeString( + 10, + f + ); + } f = /** @type {!proto.ml_metadata.Execution.State} */ (jspb.Message.getField(message, 3)); if (f != null) { writer.writeEnum( @@ -3323,6 +3735,14 @@ proto.ml_metadata.Execution.serializeBinaryToWriter = function(message, writer) f ); } + f = message.getSystemMetadata(); + if (f != null) { + writer.writeMessage( + 11, + f, + google_protobuf_any_pb.Any.serializeBinaryToWriter + ); + } }; @@ -3483,6 +3903,42 @@ proto.ml_metadata.Execution.prototype.hasType = function() { }; +/** + * optional string external_id = 10; + * @return {string} + */ +proto.ml_metadata.Execution.prototype.getExternalId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 10, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.Execution} returns this + */ +proto.ml_metadata.Execution.prototype.setExternalId = function(value) { + return jspb.Message.setField(this, 10, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.Execution} returns this + */ +proto.ml_metadata.Execution.prototype.clearExternalId = function() { + return jspb.Message.setField(this, 10, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Execution.prototype.hasExternalId = function() { + return jspb.Message.getField(this, 10) != null; +}; + + /** * optional State last_known_state = 3; * @return {!proto.ml_metadata.Execution.State} @@ -3635,6 +4091,43 @@ proto.ml_metadata.Execution.prototype.hasLastUpdateTimeSinceEpoch = function() { }; +/** + * optional google.protobuf.Any system_metadata = 11; + * @return {?proto.google.protobuf.Any} + */ +proto.ml_metadata.Execution.prototype.getSystemMetadata = function() { + return /** @type{?proto.google.protobuf.Any} */ ( + jspb.Message.getWrapperField(this, google_protobuf_any_pb.Any, 11)); +}; + + +/** + * @param {?proto.google.protobuf.Any|undefined} value + * @return {!proto.ml_metadata.Execution} returns this +*/ +proto.ml_metadata.Execution.prototype.setSystemMetadata = function(value) { + return jspb.Message.setWrapperField(this, 11, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.Execution} returns this + */ +proto.ml_metadata.Execution.prototype.clearSystemMetadata = function() { + return this.setSystemMetadata(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Execution.prototype.hasSystemMetadata = function() { + return jspb.Message.getField(this, 11) != null; +}; + + @@ -3671,6 +4164,7 @@ proto.ml_metadata.ExecutionType.toObject = function(includeInstance, msg) { name: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, version: (f = jspb.Message.getField(msg, 6)) == null ? undefined : f, description: (f = jspb.Message.getField(msg, 7)) == null ? undefined : f, + externalId: (f = jspb.Message.getField(msg, 9)) == null ? undefined : f, propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, undefined) : [], inputType: (f = msg.getInputType()) && proto.ml_metadata.ArtifactStructType.toObject(includeInstance, f), outputType: (f = msg.getOutputType()) && proto.ml_metadata.ArtifactStructType.toObject(includeInstance, f), @@ -3727,6 +4221,10 @@ proto.ml_metadata.ExecutionType.deserializeBinaryFromReader = function(msg, read var value = /** @type {string} */ (reader.readString()); msg.setDescription(value); break; + case 9: + var value = /** @type {string} */ (reader.readString()); + msg.setExternalId(value); + break; case 3: var value = msg.getPropertiesMap(); reader.readMessage(value, function(message, reader) { @@ -3804,6 +4302,13 @@ proto.ml_metadata.ExecutionType.serializeBinaryToWriter = function(message, writ f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 9)); + if (f != null) { + writer.writeString( + 9, + f + ); + } f = message.getPropertiesMap(true); if (f && f.getLength() > 0) { f.serializeBinary(3, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeEnum); @@ -3990,6 +4495,42 @@ proto.ml_metadata.ExecutionType.prototype.hasDescription = function() { }; +/** + * optional string external_id = 9; + * @return {string} + */ +proto.ml_metadata.ExecutionType.prototype.getExternalId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 9, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.ExecutionType} returns this + */ +proto.ml_metadata.ExecutionType.prototype.setExternalId = function(value) { + return jspb.Message.setField(this, 9, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ExecutionType} returns this + */ +proto.ml_metadata.ExecutionType.prototype.clearExternalId = function() { + return jspb.Message.setField(this, 9, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ExecutionType.prototype.hasExternalId = function() { + return jspb.Message.getField(this, 9) != null; +}; + + /** * map properties = 3; * @param {boolean=} opt_noLazyCreate Do not create the map if @@ -4158,6 +4699,7 @@ proto.ml_metadata.ContextType.toObject = function(includeInstance, msg) { name: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, version: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f, description: (f = jspb.Message.getField(msg, 5)) == null ? undefined : f, + externalId: (f = jspb.Message.getField(msg, 7)) == null ? undefined : f, propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, undefined) : [], baseType: (f = jspb.Message.getField(msg, 6)) == null ? undefined : f }; @@ -4212,6 +4754,10 @@ proto.ml_metadata.ContextType.deserializeBinaryFromReader = function(msg, reader var value = /** @type {string} */ (reader.readString()); msg.setDescription(value); break; + case 7: + var value = /** @type {string} */ (reader.readString()); + msg.setExternalId(value); + break; case 3: var value = msg.getPropertiesMap(); reader.readMessage(value, function(message, reader) { @@ -4279,6 +4825,13 @@ proto.ml_metadata.ContextType.serializeBinaryToWriter = function(message, writer f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 7)); + if (f != null) { + writer.writeString( + 7, + f + ); + } f = message.getPropertiesMap(true); if (f && f.getLength() > 0) { f.serializeBinary(3, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeEnum); @@ -4444,6 +4997,42 @@ proto.ml_metadata.ContextType.prototype.hasDescription = function() { }; +/** + * optional string external_id = 7; + * @return {string} + */ +proto.ml_metadata.ContextType.prototype.getExternalId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 7, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.ContextType} returns this + */ +proto.ml_metadata.ContextType.prototype.setExternalId = function(value) { + return jspb.Message.setField(this, 7, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ContextType} returns this + */ +proto.ml_metadata.ContextType.prototype.clearExternalId = function() { + return jspb.Message.setField(this, 7, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ContextType.prototype.hasExternalId = function() { + return jspb.Message.getField(this, 7) != null; +}; + + /** * map properties = 3; * @param {boolean=} opt_noLazyCreate Do not create the map if @@ -4538,10 +5127,12 @@ proto.ml_metadata.Context.toObject = function(includeInstance, msg) { name: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, typeId: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, type: (f = jspb.Message.getField(msg, 6)) == null ? undefined : f, + externalId: (f = jspb.Message.getField(msg, 9)) == null ? undefined : f, propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.Value.toObject) : [], customPropertiesMap: (f = msg.getCustomPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.Value.toObject) : [], createTimeSinceEpoch: (f = jspb.Message.getField(msg, 7)) == null ? undefined : f, - lastUpdateTimeSinceEpoch: (f = jspb.Message.getField(msg, 8)) == null ? undefined : f + lastUpdateTimeSinceEpoch: (f = jspb.Message.getField(msg, 8)) == null ? undefined : f, + systemMetadata: (f = msg.getSystemMetadata()) && google_protobuf_any_pb.Any.toObject(includeInstance, f) }; if (includeInstance) { @@ -4594,6 +5185,10 @@ proto.ml_metadata.Context.deserializeBinaryFromReader = function(msg, reader) { var value = /** @type {string} */ (reader.readString()); msg.setType(value); break; + case 9: + var value = /** @type {string} */ (reader.readString()); + msg.setExternalId(value); + break; case 4: var value = msg.getPropertiesMap(); reader.readMessage(value, function(message, reader) { @@ -4614,6 +5209,11 @@ proto.ml_metadata.Context.deserializeBinaryFromReader = function(msg, reader) { var value = /** @type {number} */ (reader.readInt64()); msg.setLastUpdateTimeSinceEpoch(value); break; + case 10: + var value = new google_protobuf_any_pb.Any; + reader.readMessage(value,google_protobuf_any_pb.Any.deserializeBinaryFromReader); + msg.setSystemMetadata(value); + break; default: reader.skipField(); break; @@ -4671,6 +5271,13 @@ proto.ml_metadata.Context.serializeBinaryToWriter = function(message, writer) { f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 9)); + if (f != null) { + writer.writeString( + 9, + f + ); + } f = message.getPropertiesMap(true); if (f && f.getLength() > 0) { f.serializeBinary(4, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeMessage, proto.ml_metadata.Value.serializeBinaryToWriter); @@ -4693,6 +5300,14 @@ proto.ml_metadata.Context.serializeBinaryToWriter = function(message, writer) { f ); } + f = message.getSystemMetadata(); + if (f != null) { + writer.writeMessage( + 10, + f, + google_protobuf_any_pb.Any.serializeBinaryToWriter + ); + } }; @@ -4840,6 +5455,42 @@ proto.ml_metadata.Context.prototype.hasType = function() { }; +/** + * optional string external_id = 9; + * @return {string} + */ +proto.ml_metadata.Context.prototype.getExternalId = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 9, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.Context} returns this + */ +proto.ml_metadata.Context.prototype.setExternalId = function(value) { + return jspb.Message.setField(this, 9, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.Context} returns this + */ +proto.ml_metadata.Context.prototype.clearExternalId = function() { + return jspb.Message.setField(this, 9, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Context.prototype.hasExternalId = function() { + return jspb.Message.getField(this, 9) != null; +}; + + /** * map properties = 4; * @param {boolean=} opt_noLazyCreate Do not create the map if @@ -4956,24 +5607,61 @@ proto.ml_metadata.Context.prototype.hasLastUpdateTimeSinceEpoch = function() { }; +/** + * optional google.protobuf.Any system_metadata = 10; + * @return {?proto.google.protobuf.Any} + */ +proto.ml_metadata.Context.prototype.getSystemMetadata = function() { + return /** @type{?proto.google.protobuf.Any} */ ( + jspb.Message.getWrapperField(this, google_protobuf_any_pb.Any, 10)); +}; + +/** + * @param {?proto.google.protobuf.Any|undefined} value + * @return {!proto.ml_metadata.Context} returns this +*/ +proto.ml_metadata.Context.prototype.setSystemMetadata = function(value) { + return jspb.Message.setWrapperField(this, 10, value); +}; -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.Context} returns this */ -proto.ml_metadata.Attribution.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.Attribution.toObject(opt_includeInstance, this); +proto.ml_metadata.Context.prototype.clearSystemMetadata = function() { + return this.setSystemMetadata(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.Context.prototype.hasSystemMetadata = function() { + return jspb.Message.getField(this, 10) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.Attribution.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.Attribution.toObject(opt_includeInstance, this); }; @@ -7915,7 +8603,7 @@ proto.ml_metadata.MySQLDatabaseConfig.deserializeBinaryFromReader = function(msg msg.setHost(value); break; case 2: - var value = /** @type {number} */ (reader.readUint32()); + var value = /** @type {number} */ (reader.readInt64()); msg.setPort(value); break; case 3: @@ -7981,7 +8669,7 @@ proto.ml_metadata.MySQLDatabaseConfig.serializeBinaryToWriter = function(message } f = /** @type {number} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeUint32( + writer.writeInt64( 2, f ); @@ -8457,7 +9145,7 @@ proto.ml_metadata.MySQLDatabaseConfig.prototype.hasHost = function() { /** - * optional uint32 port = 2; + * optional int64 port = 2; * @return {number} */ proto.ml_metadata.MySQLDatabaseConfig.prototype.getPort = function() { @@ -8931,8 +9619,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.MigrationOptions.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.MigrationOptions.toObject(opt_includeInstance, this); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PostgreSQLDatabaseConfig.toObject(opt_includeInstance, this); }; @@ -8941,14 +9629,21 @@ proto.ml_metadata.MigrationOptions.prototype.toObject = function(opt_includeInst * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.MigrationOptions} msg The msg instance to transform. + * @param {!proto.ml_metadata.PostgreSQLDatabaseConfig} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MigrationOptions.toObject = function(includeInstance, msg) { +proto.ml_metadata.PostgreSQLDatabaseConfig.toObject = function(includeInstance, msg) { var f, obj = { - enableUpgradeMigration: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, - downgradeToSchemaVersion: jspb.Message.getFieldWithDefault(msg, 2, -1) + host: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + hostaddr: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + port: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + user: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f, + password: (f = jspb.Message.getField(msg, 5)) == null ? undefined : f, + passfile: (f = jspb.Message.getField(msg, 6)) == null ? undefined : f, + dbname: (f = jspb.Message.getField(msg, 7)) == null ? undefined : f, + skipDbCreation: (f = jspb.Message.getBooleanField(msg, 8)) == null ? undefined : f, + ssloption: (f = msg.getSsloption()) && proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -8962,36 +9657,65 @@ proto.ml_metadata.MigrationOptions.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.MigrationOptions} + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} */ -proto.ml_metadata.MigrationOptions.deserializeBinary = function(bytes) { +proto.ml_metadata.PostgreSQLDatabaseConfig.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.MigrationOptions; - return proto.ml_metadata.MigrationOptions.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PostgreSQLDatabaseConfig; + return proto.ml_metadata.PostgreSQLDatabaseConfig.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.MigrationOptions} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PostgreSQLDatabaseConfig} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.MigrationOptions} + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} */ -proto.ml_metadata.MigrationOptions.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PostgreSQLDatabaseConfig.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setHost(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setHostaddr(value); + break; case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setPort(value); + break; + case 4: + var value = /** @type {string} */ (reader.readString()); + msg.setUser(value); + break; + case 5: + var value = /** @type {string} */ (reader.readString()); + msg.setPassword(value); + break; + case 6: + var value = /** @type {string} */ (reader.readString()); + msg.setPassfile(value); + break; + case 7: + var value = /** @type {string} */ (reader.readString()); + msg.setDbname(value); + break; + case 8: var value = /** @type {boolean} */ (reader.readBool()); - msg.setEnableUpgradeMigration(value); + msg.setSkipDbCreation(value); break; - case 2: - var value = /** @type {number} */ (reader.readInt64()); - msg.setDowngradeToSchemaVersion(value); + case 9: + var value = new proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions; + reader.readMessage(value,proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.deserializeBinaryFromReader); + msg.setSsloption(value); break; default: reader.skipField(); @@ -9006,9 +9730,9 @@ proto.ml_metadata.MigrationOptions.deserializeBinaryFromReader = function(msg, r * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.MigrationOptions.prototype.serializeBinary = function() { +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.MigrationOptions.serializeBinaryToWriter(this, writer); + proto.ml_metadata.PostgreSQLDatabaseConfig.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -9016,98 +9740,76 @@ proto.ml_metadata.MigrationOptions.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.MigrationOptions} message + * @param {!proto.ml_metadata.PostgreSQLDatabaseConfig} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MigrationOptions.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PostgreSQLDatabaseConfig.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeBool( - 3, + writer.writeString( + 1, f ); } - f = /** @type {number} */ (jspb.Message.getField(message, 2)); + f = /** @type {string} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeInt64( + writer.writeString( 2, f ); } -}; - - -/** - * optional bool enable_upgrade_migration = 3; - * @return {boolean} - */ -proto.ml_metadata.MigrationOptions.prototype.getEnableUpgradeMigration = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); -}; - - -/** - * @param {boolean} value - * @return {!proto.ml_metadata.MigrationOptions} returns this - */ -proto.ml_metadata.MigrationOptions.prototype.setEnableUpgradeMigration = function(value) { - return jspb.Message.setField(this, 3, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MigrationOptions} returns this - */ -proto.ml_metadata.MigrationOptions.prototype.clearEnableUpgradeMigration = function() { - return jspb.Message.setField(this, 3, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.MigrationOptions.prototype.hasEnableUpgradeMigration = function() { - return jspb.Message.getField(this, 3) != null; -}; - - -/** - * optional int64 downgrade_to_schema_version = 2; - * @return {number} - */ -proto.ml_metadata.MigrationOptions.prototype.getDowngradeToSchemaVersion = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, -1)); -}; - - -/** - * @param {number} value - * @return {!proto.ml_metadata.MigrationOptions} returns this - */ -proto.ml_metadata.MigrationOptions.prototype.setDowngradeToSchemaVersion = function(value) { - return jspb.Message.setField(this, 2, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MigrationOptions} returns this - */ -proto.ml_metadata.MigrationOptions.prototype.clearDowngradeToSchemaVersion = function() { - return jspb.Message.setField(this, 2, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.MigrationOptions.prototype.hasDowngradeToSchemaVersion = function() { - return jspb.Message.getField(this, 2) != null; + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeString( + 4, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeString( + 5, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 6)); + if (f != null) { + writer.writeString( + 6, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 7)); + if (f != null) { + writer.writeString( + 7, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 8)); + if (f != null) { + writer.writeBool( + 8, + f + ); + } + f = message.getSsloption(); + if (f != null) { + writer.writeMessage( + 9, + f, + proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.serializeBinaryToWriter + ); + } }; @@ -9127,8 +9829,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.RetryOptions.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.RetryOptions.toObject(opt_includeInstance, this); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.toObject(opt_includeInstance, this); }; @@ -9137,13 +9839,17 @@ proto.ml_metadata.RetryOptions.prototype.toObject = function(opt_includeInstance * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.RetryOptions} msg The msg instance to transform. + * @param {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.RetryOptions.toObject = function(includeInstance, msg) { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.toObject = function(includeInstance, msg) { var f, obj = { - maxNumRetries: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + sslmode: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + sslcert: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + sslkey: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + sslpassword: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f, + sslrootcert: (f = jspb.Message.getField(msg, 5)) == null ? undefined : f }; if (includeInstance) { @@ -9157,23 +9863,23 @@ proto.ml_metadata.RetryOptions.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.RetryOptions} + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} */ -proto.ml_metadata.RetryOptions.deserializeBinary = function(bytes) { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.RetryOptions; - return proto.ml_metadata.RetryOptions.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions; + return proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.RetryOptions} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.RetryOptions} + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} */ -proto.ml_metadata.RetryOptions.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -9181,8 +9887,24 @@ proto.ml_metadata.RetryOptions.deserializeBinaryFromReader = function(msg, reade var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setMaxNumRetries(value); + var value = /** @type {string} */ (reader.readString()); + msg.setSslmode(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setSslcert(value); + break; + case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setSslkey(value); + break; + case 4: + var value = /** @type {string} */ (reader.readString()); + msg.setSslpassword(value); + break; + case 5: + var value = /** @type {string} */ (reader.readString()); + msg.setSslrootcert(value); break; default: reader.skipField(); @@ -9197,9 +9919,9 @@ proto.ml_metadata.RetryOptions.deserializeBinaryFromReader = function(msg, reade * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.RetryOptions.prototype.serializeBinary = function() { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.RetryOptions.serializeBinaryToWriter(this, writer); + proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -9207,45 +9929,73 @@ proto.ml_metadata.RetryOptions.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.RetryOptions} message + * @param {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.RetryOptions.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeInt64( + writer.writeString( 1, f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeString( + 4, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeString( + 5, + f + ); + } }; /** - * optional int64 max_num_retries = 1; - * @return {number} - */ -proto.ml_metadata.RetryOptions.prototype.getMaxNumRetries = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); -}; - - -/** - * @param {number} value - * @return {!proto.ml_metadata.RetryOptions} returns this + * optional string sslmode = 1; + * @return {string} */ -proto.ml_metadata.RetryOptions.prototype.setMaxNumRetries = function(value) { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.getSslmode = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.setSslmode = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.RetryOptions} returns this + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this */ -proto.ml_metadata.RetryOptions.prototype.clearMaxNumRetries = function() { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.clearSslmode = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -9254,219 +10004,107 @@ proto.ml_metadata.RetryOptions.prototype.clearMaxNumRetries = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.RetryOptions.prototype.hasMaxNumRetries = function() { +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.hasSslmode = function() { return jspb.Message.getField(this, 1) != null; }; - -/** - * Oneof group definitions for this message. Each group defines the field - * numbers belonging to that group. When of these fields' value is set, all - * other fields in the group are cleared. During deserialization, if multiple - * fields are encountered for a group, only the last value seen will be kept. - * @private {!Array>} - * @const - */ -proto.ml_metadata.ConnectionConfig.oneofGroups_ = [[1,2,3]]; - /** - * @enum {number} + * optional string sslcert = 2; + * @return {string} */ -proto.ml_metadata.ConnectionConfig.ConfigCase = { - CONFIG_NOT_SET: 0, - FAKE_DATABASE: 1, - MYSQL: 2, - SQLITE: 3 +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.getSslcert = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; + /** - * @return {proto.ml_metadata.ConnectionConfig.ConfigCase} + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this */ -proto.ml_metadata.ConnectionConfig.prototype.getConfigCase = function() { - return /** @type {proto.ml_metadata.ConnectionConfig.ConfigCase} */(jspb.Message.computeOneofCase(this, proto.ml_metadata.ConnectionConfig.oneofGroups_[0])); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.setSslcert = function(value) { + return jspb.Message.setField(this, 2, value); }; - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this */ -proto.ml_metadata.ConnectionConfig.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ConnectionConfig.toObject(opt_includeInstance, this); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.clearSslcert = function() { + return jspb.Message.setField(this, 2, undefined); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ConnectionConfig} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.ConnectionConfig.toObject = function(includeInstance, msg) { - var f, obj = { - fakeDatabase: (f = msg.getFakeDatabase()) && proto.ml_metadata.FakeDatabaseConfig.toObject(includeInstance, f), - mysql: (f = msg.getMysql()) && proto.ml_metadata.MySQLDatabaseConfig.toObject(includeInstance, f), - sqlite: (f = msg.getSqlite()) && proto.ml_metadata.SqliteMetadataSourceConfig.toObject(includeInstance, f), - retryOptions: (f = msg.getRetryOptions()) && proto.ml_metadata.RetryOptions.toObject(includeInstance, f) - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.hasSslcert = function() { + return jspb.Message.getField(this, 2) != null; }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ConnectionConfig} + * optional string sslkey = 3; + * @return {string} */ -proto.ml_metadata.ConnectionConfig.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ConnectionConfig; - return proto.ml_metadata.ConnectionConfig.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.getSslkey = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.ConnectionConfig} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ConnectionConfig} + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this */ -proto.ml_metadata.ConnectionConfig.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = new proto.ml_metadata.FakeDatabaseConfig; - reader.readMessage(value,proto.ml_metadata.FakeDatabaseConfig.deserializeBinaryFromReader); - msg.setFakeDatabase(value); - break; - case 2: - var value = new proto.ml_metadata.MySQLDatabaseConfig; - reader.readMessage(value,proto.ml_metadata.MySQLDatabaseConfig.deserializeBinaryFromReader); - msg.setMysql(value); - break; - case 3: - var value = new proto.ml_metadata.SqliteMetadataSourceConfig; - reader.readMessage(value,proto.ml_metadata.SqliteMetadataSourceConfig.deserializeBinaryFromReader); - msg.setSqlite(value); - break; - case 4: - var value = new proto.ml_metadata.RetryOptions; - reader.readMessage(value,proto.ml_metadata.RetryOptions.deserializeBinaryFromReader); - msg.setRetryOptions(value); - break; - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.setSslkey = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this */ -proto.ml_metadata.ConnectionConfig.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ConnectionConfig.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.clearSslkey = function() { + return jspb.Message.setField(this, 3, undefined); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ConnectionConfig} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.ConnectionConfig.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getFakeDatabase(); - if (f != null) { - writer.writeMessage( - 1, - f, - proto.ml_metadata.FakeDatabaseConfig.serializeBinaryToWriter - ); - } - f = message.getMysql(); - if (f != null) { - writer.writeMessage( - 2, - f, - proto.ml_metadata.MySQLDatabaseConfig.serializeBinaryToWriter - ); - } - f = message.getSqlite(); - if (f != null) { - writer.writeMessage( - 3, - f, - proto.ml_metadata.SqliteMetadataSourceConfig.serializeBinaryToWriter - ); - } - f = message.getRetryOptions(); - if (f != null) { - writer.writeMessage( - 4, - f, - proto.ml_metadata.RetryOptions.serializeBinaryToWriter - ); - } +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.hasSslkey = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * optional FakeDatabaseConfig fake_database = 1; - * @return {?proto.ml_metadata.FakeDatabaseConfig} + * optional string sslpassword = 4; + * @return {string} */ -proto.ml_metadata.ConnectionConfig.prototype.getFakeDatabase = function() { - return /** @type{?proto.ml_metadata.FakeDatabaseConfig} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.FakeDatabaseConfig, 1)); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.getSslpassword = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, "")); }; /** - * @param {?proto.ml_metadata.FakeDatabaseConfig|undefined} value - * @return {!proto.ml_metadata.ConnectionConfig} returns this -*/ -proto.ml_metadata.ConnectionConfig.prototype.setFakeDatabase = function(value) { - return jspb.Message.setOneofWrapperField(this, 1, proto.ml_metadata.ConnectionConfig.oneofGroups_[0], value); + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.setSslpassword = function(value) { + return jspb.Message.setField(this, 4, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ConnectionConfig} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this */ -proto.ml_metadata.ConnectionConfig.prototype.clearFakeDatabase = function() { - return this.setFakeDatabase(undefined); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.clearSslpassword = function() { + return jspb.Message.setField(this, 4, undefined); }; @@ -9474,36 +10112,35 @@ proto.ml_metadata.ConnectionConfig.prototype.clearFakeDatabase = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ConnectionConfig.prototype.hasFakeDatabase = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.hasSslpassword = function() { + return jspb.Message.getField(this, 4) != null; }; /** - * optional MySQLDatabaseConfig mysql = 2; - * @return {?proto.ml_metadata.MySQLDatabaseConfig} + * optional string sslrootcert = 5; + * @return {string} */ -proto.ml_metadata.ConnectionConfig.prototype.getMysql = function() { - return /** @type{?proto.ml_metadata.MySQLDatabaseConfig} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.MySQLDatabaseConfig, 2)); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.getSslrootcert = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 5, "")); }; /** - * @param {?proto.ml_metadata.MySQLDatabaseConfig|undefined} value - * @return {!proto.ml_metadata.ConnectionConfig} returns this -*/ -proto.ml_metadata.ConnectionConfig.prototype.setMysql = function(value) { - return jspb.Message.setOneofWrapperField(this, 2, proto.ml_metadata.ConnectionConfig.oneofGroups_[0], value); + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.setSslrootcert = function(value) { + return jspb.Message.setField(this, 5, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ConnectionConfig} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} returns this */ -proto.ml_metadata.ConnectionConfig.prototype.clearMysql = function() { - return this.setMysql(undefined); +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.clearSslrootcert = function() { + return jspb.Message.setField(this, 5, undefined); }; @@ -9511,36 +10148,35 @@ proto.ml_metadata.ConnectionConfig.prototype.clearMysql = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ConnectionConfig.prototype.hasMysql = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions.prototype.hasSslrootcert = function() { + return jspb.Message.getField(this, 5) != null; }; /** - * optional SqliteMetadataSourceConfig sqlite = 3; - * @return {?proto.ml_metadata.SqliteMetadataSourceConfig} + * optional string host = 1; + * @return {string} */ -proto.ml_metadata.ConnectionConfig.prototype.getSqlite = function() { - return /** @type{?proto.ml_metadata.SqliteMetadataSourceConfig} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.SqliteMetadataSourceConfig, 3)); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getHost = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {?proto.ml_metadata.SqliteMetadataSourceConfig|undefined} value - * @return {!proto.ml_metadata.ConnectionConfig} returns this -*/ -proto.ml_metadata.ConnectionConfig.prototype.setSqlite = function(value) { - return jspb.Message.setOneofWrapperField(this, 3, proto.ml_metadata.ConnectionConfig.oneofGroups_[0], value); + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setHost = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ConnectionConfig} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.ConnectionConfig.prototype.clearSqlite = function() { - return this.setSqlite(undefined); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearHost = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -9548,36 +10184,35 @@ proto.ml_metadata.ConnectionConfig.prototype.clearSqlite = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ConnectionConfig.prototype.hasSqlite = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasHost = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional RetryOptions retry_options = 4; - * @return {?proto.ml_metadata.RetryOptions} + * optional string hostaddr = 2; + * @return {string} */ -proto.ml_metadata.ConnectionConfig.prototype.getRetryOptions = function() { - return /** @type{?proto.ml_metadata.RetryOptions} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.RetryOptions, 4)); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getHostaddr = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {?proto.ml_metadata.RetryOptions|undefined} value - * @return {!proto.ml_metadata.ConnectionConfig} returns this -*/ -proto.ml_metadata.ConnectionConfig.prototype.setRetryOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setHostaddr = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ConnectionConfig} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.ConnectionConfig.prototype.clearRetryOptions = function() { - return this.setRetryOptions(undefined); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearHostaddr = function() { + return jspb.Message.setField(this, 2, undefined); }; @@ -9585,159 +10220,107 @@ proto.ml_metadata.ConnectionConfig.prototype.clearRetryOptions = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ConnectionConfig.prototype.hasRetryOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasHostaddr = function() { + return jspb.Message.getField(this, 2) != null; }; +/** + * optional string port = 3; + * @return {string} + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getPort = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +}; - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GrpcChannelArguments.toObject(opt_includeInstance, this); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setPort = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GrpcChannelArguments} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.toObject = function(includeInstance, msg) { - var f, obj = { - maxReceiveMessageLength: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - http2MaxPingStrikes: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f - }; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearPort = function() { + return jspb.Message.setField(this, 3, undefined); +}; - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasPort = function() { + return jspb.Message.getField(this, 3) != null; }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GrpcChannelArguments} + * optional string user = 4; + * @return {string} */ -proto.ml_metadata.GrpcChannelArguments.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GrpcChannelArguments; - return proto.ml_metadata.GrpcChannelArguments.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getUser = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, "")); }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.GrpcChannelArguments} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GrpcChannelArguments} + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setMaxReceiveMessageLength(value); - break; - case 2: - var value = /** @type {number} */ (reader.readInt64()); - msg.setHttp2MaxPingStrikes(value); - break; - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setUser = function(value) { + return jspb.Message.setField(this, 4, value); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GrpcChannelArguments.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearUser = function() { + return jspb.Message.setField(this, 4, undefined); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GrpcChannelArguments} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GrpcChannelArguments.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( - 1, - f - ); - } - f = /** @type {number} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeInt64( - 2, - f - ); - } +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasUser = function() { + return jspb.Message.getField(this, 4) != null; }; /** - * optional int64 max_receive_message_length = 1; - * @return {number} + * optional string password = 5; + * @return {string} */ -proto.ml_metadata.GrpcChannelArguments.prototype.getMaxReceiveMessageLength = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getPassword = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 5, "")); }; /** - * @param {number} value - * @return {!proto.ml_metadata.GrpcChannelArguments} returns this + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.prototype.setMaxReceiveMessageLength = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setPassword = function(value) { + return jspb.Message.setField(this, 5, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GrpcChannelArguments} returns this + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.prototype.clearMaxReceiveMessageLength = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearPassword = function() { + return jspb.Message.setField(this, 5, undefined); }; @@ -9745,35 +10328,35 @@ proto.ml_metadata.GrpcChannelArguments.prototype.clearMaxReceiveMessageLength = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GrpcChannelArguments.prototype.hasMaxReceiveMessageLength = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasPassword = function() { + return jspb.Message.getField(this, 5) != null; }; /** - * optional int64 http2_max_ping_strikes = 2; - * @return {number} + * optional string passfile = 6; + * @return {string} */ -proto.ml_metadata.GrpcChannelArguments.prototype.getHttp2MaxPingStrikes = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getPassfile = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 6, "")); }; /** - * @param {number} value - * @return {!proto.ml_metadata.GrpcChannelArguments} returns this + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.prototype.setHttp2MaxPingStrikes = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setPassfile = function(value) { + return jspb.Message.setField(this, 6, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GrpcChannelArguments} returns this + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.GrpcChannelArguments.prototype.clearHttp2MaxPingStrikes = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearPassfile = function() { + return jspb.Message.setField(this, 6, undefined); }; @@ -9781,172 +10364,117 @@ proto.ml_metadata.GrpcChannelArguments.prototype.clearHttp2MaxPingStrikes = func * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GrpcChannelArguments.prototype.hasHttp2MaxPingStrikes = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasPassfile = function() { + return jspb.Message.getField(this, 6) != null; }; +/** + * optional string dbname = 7; + * @return {string} + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getDbname = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 7, "")); +}; - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * @param {string} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.MetadataStoreClientConfig.toObject(opt_includeInstance, this); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setDbname = function(value) { + return jspb.Message.setField(this, 7, value); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.MetadataStoreClientConfig} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.toObject = function(includeInstance, msg) { - var f, obj = { - host: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - port: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - sslConfig: (f = msg.getSslConfig()) && proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.toObject(includeInstance, f), - channelArguments: (f = msg.getChannelArguments()) && proto.ml_metadata.GrpcChannelArguments.toObject(includeInstance, f), - clientTimeoutSec: (f = jspb.Message.getOptionalFloatingPointField(msg, 5)) == null ? undefined : f - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearDbname = function() { + return jspb.Message.setField(this, 7, undefined); }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.MetadataStoreClientConfig} + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.MetadataStoreClientConfig.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.MetadataStoreClientConfig; - return proto.ml_metadata.MetadataStoreClientConfig.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasDbname = function() { + return jspb.Message.getField(this, 7) != null; }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.MetadataStoreClientConfig} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.MetadataStoreClientConfig} + * optional bool skip_db_creation = 8; + * @return {boolean} */ -proto.ml_metadata.MetadataStoreClientConfig.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setHost(value); - break; - case 2: - var value = /** @type {number} */ (reader.readUint32()); - msg.setPort(value); - break; - case 3: - var value = new proto.ml_metadata.MetadataStoreClientConfig.SSLConfig; - reader.readMessage(value,proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinaryFromReader); - msg.setSslConfig(value); - break; - case 4: - var value = new proto.ml_metadata.GrpcChannelArguments; - reader.readMessage(value,proto.ml_metadata.GrpcChannelArguments.deserializeBinaryFromReader); - msg.setChannelArguments(value); - break; - case 5: - var value = /** @type {number} */ (reader.readDouble()); - msg.setClientTimeoutSec(value); - break; - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getSkipDbCreation = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 8, false)); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * @param {boolean} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.MetadataStoreClientConfig.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setSkipDbCreation = function(value) { + return jspb.Message.setField(this, 8, value); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.MetadataStoreClientConfig} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( - 1, - f - ); - } - f = /** @type {number} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeUint32( - 2, - f - ); - } - f = message.getSslConfig(); - if (f != null) { - writer.writeMessage( - 3, - f, - proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.serializeBinaryToWriter - ); - } - f = message.getChannelArguments(); - if (f != null) { - writer.writeMessage( - 4, - f, - proto.ml_metadata.GrpcChannelArguments.serializeBinaryToWriter - ); - } - f = /** @type {number} */ (jspb.Message.getField(message, 5)); - if (f != null) { - writer.writeDouble( - 5, - f - ); - } +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearSkipDbCreation = function() { + return jspb.Message.setField(this, 8, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasSkipDbCreation = function() { + return jspb.Message.getField(this, 8) != null; +}; + + +/** + * optional SSLOptions ssloption = 9; + * @return {?proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.getSsloption = function() { + return /** @type{?proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions, 9)); +}; + + +/** + * @param {?proto.ml_metadata.PostgreSQLDatabaseConfig.SSLOptions|undefined} value + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this +*/ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.setSsloption = function(value) { + return jspb.Message.setWrapperField(this, 9, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PostgreSQLDatabaseConfig} returns this + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.clearSsloption = function() { + return this.setSsloption(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PostgreSQLDatabaseConfig.prototype.hasSsloption = function() { + return jspb.Message.getField(this, 9) != null; }; @@ -9966,8 +10494,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.toObject(opt_includeInstance, this); +proto.ml_metadata.MigrationOptions.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.MigrationOptions.toObject(opt_includeInstance, this); }; @@ -9976,15 +10504,14 @@ proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.toObject = funct * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} msg The msg instance to transform. + * @param {!proto.ml_metadata.MigrationOptions} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.toObject = function(includeInstance, msg) { +proto.ml_metadata.MigrationOptions.toObject = function(includeInstance, msg) { var f, obj = { - clientKey: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - serverCert: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - customCa: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f + enableUpgradeMigration: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, + downgradeToSchemaVersion: jspb.Message.getFieldWithDefault(msg, 2, -1) }; if (includeInstance) { @@ -9998,40 +10525,36 @@ proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.toObject = function(includ /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} + * @return {!proto.ml_metadata.MigrationOptions} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinary = function(bytes) { +proto.ml_metadata.MigrationOptions.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.MetadataStoreClientConfig.SSLConfig; - return proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.MigrationOptions; + return proto.ml_metadata.MigrationOptions.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} msg The message object to deserialize into. + * @param {!proto.ml_metadata.MigrationOptions} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} + * @return {!proto.ml_metadata.MigrationOptions} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.MigrationOptions.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { - case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setClientKey(value); + case 3: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setEnableUpgradeMigration(value); break; case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setServerCert(value); - break; - case 3: - var value = /** @type {string} */ (reader.readString()); - msg.setCustomCa(value); + var value = /** @type {number} */ (reader.readInt64()); + msg.setDowngradeToSchemaVersion(value); break; default: reader.skipField(); @@ -10046,9 +10569,9 @@ proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinaryFromReade * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.serializeBinary = function() { +proto.ml_metadata.MigrationOptions.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.serializeBinaryToWriter(this, writer); + proto.ml_metadata.MigrationOptions.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -10056,60 +10579,53 @@ proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.serializeBinary /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} message + * @param {!proto.ml_metadata.MigrationOptions} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.MigrationOptions.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); + f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); if (f != null) { - writer.writeString( - 1, + writer.writeBool( + 3, f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); + f = /** @type {number} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeString( + writer.writeInt64( 2, f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeString( - 3, - f - ); - } }; /** - * optional string client_key = 1; - * @return {string} + * optional bool enable_upgrade_migration = 3; + * @return {boolean} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.getClientKey = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +proto.ml_metadata.MigrationOptions.prototype.getEnableUpgradeMigration = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this + * @param {boolean} value + * @return {!proto.ml_metadata.MigrationOptions} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.setClientKey = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.MigrationOptions.prototype.setEnableUpgradeMigration = function(value) { + return jspb.Message.setField(this, 3, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this + * @return {!proto.ml_metadata.MigrationOptions} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearClientKey = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.MigrationOptions.prototype.clearEnableUpgradeMigration = function() { + return jspb.Message.setField(this, 3, undefined); }; @@ -10117,34 +10633,34 @@ proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearClientKey = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.hasClientKey = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.MigrationOptions.prototype.hasEnableUpgradeMigration = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * optional string server_cert = 2; - * @return {string} + * optional int64 downgrade_to_schema_version = 2; + * @return {number} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.getServerCert = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.MigrationOptions.prototype.getDowngradeToSchemaVersion = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, -1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this + * @param {number} value + * @return {!proto.ml_metadata.MigrationOptions} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.setServerCert = function(value) { +proto.ml_metadata.MigrationOptions.prototype.setDowngradeToSchemaVersion = function(value) { return jspb.Message.setField(this, 2, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this + * @return {!proto.ml_metadata.MigrationOptions} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearServerCert = function() { +proto.ml_metadata.MigrationOptions.prototype.clearDowngradeToSchemaVersion = function() { return jspb.Message.setField(this, 2, undefined); }; @@ -10153,181 +10669,147 @@ proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearServerCert * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.hasServerCert = function() { +proto.ml_metadata.MigrationOptions.prototype.hasDowngradeToSchemaVersion = function() { return jspb.Message.getField(this, 2) != null; }; -/** - * optional string custom_ca = 3; - * @return {string} - */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.getCustomCa = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); -}; - -/** - * @param {string} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this - */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.setCustomCa = function(value) { - return jspb.Message.setField(this, 3, value); -}; +if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this - */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearCustomCa = function() { - return jspb.Message.setField(this, 3, undefined); + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.RetryOptions.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.RetryOptions.toObject(opt_includeInstance, this); }; /** - * Returns whether this field is set. - * @return {boolean} + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.RetryOptions} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.hasCustomCa = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.RetryOptions.toObject = function(includeInstance, msg) { + var f, obj = { + maxNumRetries: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; }; +} /** - * optional string host = 1; - * @return {string} + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.RetryOptions} */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.getHost = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +proto.ml_metadata.RetryOptions.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.RetryOptions; + return proto.ml_metadata.RetryOptions.deserializeBinaryFromReader(msg, reader); }; /** - * @param {string} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.RetryOptions} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.RetryOptions} */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.setHost = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.RetryOptions.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt64()); + msg.setMaxNumRetries(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.clearHost = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.RetryOptions.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.RetryOptions.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * Returns whether this field is set. - * @return {boolean} + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.RetryOptions} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.hasHost = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.RetryOptions.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( + 1, + f + ); + } }; /** - * optional uint32 port = 2; + * optional int64 max_num_retries = 1; * @return {number} */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.getPort = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +proto.ml_metadata.RetryOptions.prototype.getMaxNumRetries = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** * @param {number} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + * @return {!proto.ml_metadata.RetryOptions} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.setPort = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.RetryOptions.prototype.setMaxNumRetries = function(value) { + return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this - */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.clearPort = function() { - return jspb.Message.setField(this, 2, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.hasPort = function() { - return jspb.Message.getField(this, 2) != null; -}; - - -/** - * optional SSLConfig ssl_config = 3; - * @return {?proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} - */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.getSslConfig = function() { - return /** @type{?proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.MetadataStoreClientConfig.SSLConfig, 3)); -}; - - -/** - * @param {?proto.ml_metadata.MetadataStoreClientConfig.SSLConfig|undefined} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this -*/ -proto.ml_metadata.MetadataStoreClientConfig.prototype.setSslConfig = function(value) { - return jspb.Message.setWrapperField(this, 3, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this - */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.clearSslConfig = function() { - return this.setSslConfig(undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.hasSslConfig = function() { - return jspb.Message.getField(this, 3) != null; -}; - - -/** - * optional GrpcChannelArguments channel_arguments = 4; - * @return {?proto.ml_metadata.GrpcChannelArguments} - */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.getChannelArguments = function() { - return /** @type{?proto.ml_metadata.GrpcChannelArguments} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.GrpcChannelArguments, 4)); -}; - - -/** - * @param {?proto.ml_metadata.GrpcChannelArguments|undefined} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this -*/ -proto.ml_metadata.MetadataStoreClientConfig.prototype.setChannelArguments = function(value) { - return jspb.Message.setWrapperField(this, 4, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + * @return {!proto.ml_metadata.RetryOptions} returns this */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.clearChannelArguments = function() { - return this.setChannelArguments(undefined); +proto.ml_metadata.RetryOptions.prototype.clearMaxNumRetries = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -10335,50 +10817,42 @@ proto.ml_metadata.MetadataStoreClientConfig.prototype.clearChannelArguments = fu * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.hasChannelArguments = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.RetryOptions.prototype.hasMaxNumRetries = function() { + return jspb.Message.getField(this, 1) != null; }; -/** - * optional double client_timeout_sec = 5; - * @return {number} - */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.getClientTimeoutSec = function() { - return /** @type {number} */ (jspb.Message.getFloatingPointFieldWithDefault(this, 5, 0.0)); -}; - /** - * @param {number} value - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + * Oneof group definitions for this message. Each group defines the field + * numbers belonging to that group. When of these fields' value is set, all + * other fields in the group are cleared. During deserialization, if multiple + * fields are encountered for a group, only the last value seen will be kept. + * @private {!Array>} + * @const */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.setClientTimeoutSec = function(value) { - return jspb.Message.setField(this, 5, value); -}; - +proto.ml_metadata.ConnectionConfig.oneofGroups_ = [[1,2,3,5]]; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + * @enum {number} */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.clearClientTimeoutSec = function() { - return jspb.Message.setField(this, 5, undefined); +proto.ml_metadata.ConnectionConfig.ConfigCase = { + CONFIG_NOT_SET: 0, + FAKE_DATABASE: 1, + MYSQL: 2, + SQLITE: 3, + POSTGRESQL: 5 }; - /** - * Returns whether this field is set. - * @return {boolean} + * @return {proto.ml_metadata.ConnectionConfig.ConfigCase} */ -proto.ml_metadata.MetadataStoreClientConfig.prototype.hasClientTimeoutSec = function() { - return jspb.Message.getField(this, 5) != null; +proto.ml_metadata.ConnectionConfig.prototype.getConfigCase = function() { + return /** @type {proto.ml_metadata.ConnectionConfig.ConfigCase} */(jspb.Message.computeOneofCase(this, proto.ml_metadata.ConnectionConfig.oneofGroups_[0])); }; - - if (jspb.Message.GENERATE_TO_OBJECT) { /** * Creates an object representation of this proto. @@ -10392,8 +10866,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.MetadataStoreServerConfig.toObject(opt_includeInstance, this); +proto.ml_metadata.ConnectionConfig.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ConnectionConfig.toObject(opt_includeInstance, this); }; @@ -10402,15 +10876,17 @@ proto.ml_metadata.MetadataStoreServerConfig.prototype.toObject = function(opt_in * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.MetadataStoreServerConfig} msg The msg instance to transform. + * @param {!proto.ml_metadata.ConnectionConfig} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreServerConfig.toObject = function(includeInstance, msg) { +proto.ml_metadata.ConnectionConfig.toObject = function(includeInstance, msg) { var f, obj = { - connectionConfig: (f = msg.getConnectionConfig()) && proto.ml_metadata.ConnectionConfig.toObject(includeInstance, f), - migrationOptions: (f = msg.getMigrationOptions()) && proto.ml_metadata.MigrationOptions.toObject(includeInstance, f), - sslConfig: (f = msg.getSslConfig()) && proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.toObject(includeInstance, f) + fakeDatabase: (f = msg.getFakeDatabase()) && proto.ml_metadata.FakeDatabaseConfig.toObject(includeInstance, f), + mysql: (f = msg.getMysql()) && proto.ml_metadata.MySQLDatabaseConfig.toObject(includeInstance, f), + sqlite: (f = msg.getSqlite()) && proto.ml_metadata.SqliteMetadataSourceConfig.toObject(includeInstance, f), + postgresql: (f = msg.getPostgresql()) && proto.ml_metadata.PostgreSQLDatabaseConfig.toObject(includeInstance, f), + retryOptions: (f = msg.getRetryOptions()) && proto.ml_metadata.RetryOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -10424,23 +10900,23 @@ proto.ml_metadata.MetadataStoreServerConfig.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.MetadataStoreServerConfig} + * @return {!proto.ml_metadata.ConnectionConfig} */ -proto.ml_metadata.MetadataStoreServerConfig.deserializeBinary = function(bytes) { +proto.ml_metadata.ConnectionConfig.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.MetadataStoreServerConfig; - return proto.ml_metadata.MetadataStoreServerConfig.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.ConnectionConfig; + return proto.ml_metadata.ConnectionConfig.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.MetadataStoreServerConfig} msg The message object to deserialize into. + * @param {!proto.ml_metadata.ConnectionConfig} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.MetadataStoreServerConfig} + * @return {!proto.ml_metadata.ConnectionConfig} */ -proto.ml_metadata.MetadataStoreServerConfig.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.ConnectionConfig.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -10448,19 +10924,29 @@ proto.ml_metadata.MetadataStoreServerConfig.deserializeBinaryFromReader = functi var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new proto.ml_metadata.ConnectionConfig; - reader.readMessage(value,proto.ml_metadata.ConnectionConfig.deserializeBinaryFromReader); - msg.setConnectionConfig(value); + var value = new proto.ml_metadata.FakeDatabaseConfig; + reader.readMessage(value,proto.ml_metadata.FakeDatabaseConfig.deserializeBinaryFromReader); + msg.setFakeDatabase(value); + break; + case 2: + var value = new proto.ml_metadata.MySQLDatabaseConfig; + reader.readMessage(value,proto.ml_metadata.MySQLDatabaseConfig.deserializeBinaryFromReader); + msg.setMysql(value); break; case 3: - var value = new proto.ml_metadata.MigrationOptions; - reader.readMessage(value,proto.ml_metadata.MigrationOptions.deserializeBinaryFromReader); - msg.setMigrationOptions(value); + var value = new proto.ml_metadata.SqliteMetadataSourceConfig; + reader.readMessage(value,proto.ml_metadata.SqliteMetadataSourceConfig.deserializeBinaryFromReader); + msg.setSqlite(value); break; - case 2: - var value = new proto.ml_metadata.MetadataStoreServerConfig.SSLConfig; - reader.readMessage(value,proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinaryFromReader); - msg.setSslConfig(value); + case 5: + var value = new proto.ml_metadata.PostgreSQLDatabaseConfig; + reader.readMessage(value,proto.ml_metadata.PostgreSQLDatabaseConfig.deserializeBinaryFromReader); + msg.setPostgresql(value); + break; + case 4: + var value = new proto.ml_metadata.RetryOptions; + reader.readMessage(value,proto.ml_metadata.RetryOptions.deserializeBinaryFromReader); + msg.setRetryOptions(value); break; default: reader.skipField(); @@ -10475,9 +10961,9 @@ proto.ml_metadata.MetadataStoreServerConfig.deserializeBinaryFromReader = functi * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.serializeBinary = function() { +proto.ml_metadata.ConnectionConfig.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.MetadataStoreServerConfig.serializeBinaryToWriter(this, writer); + proto.ml_metadata.ConnectionConfig.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -10485,211 +10971,154 @@ proto.ml_metadata.MetadataStoreServerConfig.prototype.serializeBinary = function /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.MetadataStoreServerConfig} message + * @param {!proto.ml_metadata.ConnectionConfig} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreServerConfig.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.ConnectionConfig.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getConnectionConfig(); + f = message.getFakeDatabase(); if (f != null) { writer.writeMessage( 1, f, - proto.ml_metadata.ConnectionConfig.serializeBinaryToWriter + proto.ml_metadata.FakeDatabaseConfig.serializeBinaryToWriter ); } - f = message.getMigrationOptions(); + f = message.getMysql(); + if (f != null) { + writer.writeMessage( + 2, + f, + proto.ml_metadata.MySQLDatabaseConfig.serializeBinaryToWriter + ); + } + f = message.getSqlite(); if (f != null) { writer.writeMessage( 3, f, - proto.ml_metadata.MigrationOptions.serializeBinaryToWriter + proto.ml_metadata.SqliteMetadataSourceConfig.serializeBinaryToWriter ); } - f = message.getSslConfig(); + f = message.getPostgresql(); if (f != null) { writer.writeMessage( - 2, + 5, f, - proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.serializeBinaryToWriter + proto.ml_metadata.PostgreSQLDatabaseConfig.serializeBinaryToWriter + ); + } + f = message.getRetryOptions(); + if (f != null) { + writer.writeMessage( + 4, + f, + proto.ml_metadata.RetryOptions.serializeBinaryToWriter ); } }; +/** + * optional FakeDatabaseConfig fake_database = 1; + * @return {?proto.ml_metadata.FakeDatabaseConfig} + */ +proto.ml_metadata.ConnectionConfig.prototype.getFakeDatabase = function() { + return /** @type{?proto.ml_metadata.FakeDatabaseConfig} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.FakeDatabaseConfig, 1)); +}; + +/** + * @param {?proto.ml_metadata.FakeDatabaseConfig|undefined} value + * @return {!proto.ml_metadata.ConnectionConfig} returns this +*/ +proto.ml_metadata.ConnectionConfig.prototype.setFakeDatabase = function(value) { + return jspb.Message.setOneofWrapperField(this, 1, proto.ml_metadata.ConnectionConfig.oneofGroups_[0], value); +}; -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ConnectionConfig} returns this */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.toObject(opt_includeInstance, this); +proto.ml_metadata.ConnectionConfig.prototype.clearFakeDatabase = function() { + return this.setFakeDatabase(undefined); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.toObject = function(includeInstance, msg) { - var f, obj = { - serverKey: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - serverCert: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - customCa: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, - clientVerify: (f = jspb.Message.getBooleanField(msg, 4)) == null ? undefined : f - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +proto.ml_metadata.ConnectionConfig.prototype.hasFakeDatabase = function() { + return jspb.Message.getField(this, 1) != null; }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} + * optional MySQLDatabaseConfig mysql = 2; + * @return {?proto.ml_metadata.MySQLDatabaseConfig} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.MetadataStoreServerConfig.SSLConfig; - return proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.ConnectionConfig.prototype.getMysql = function() { + return /** @type{?proto.ml_metadata.MySQLDatabaseConfig} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.MySQLDatabaseConfig, 2)); }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} - */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setServerKey(value); - break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setServerCert(value); - break; - case 3: - var value = /** @type {string} */ (reader.readString()); - msg.setCustomCa(value); - break; - case 4: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setClientVerify(value); - break; - default: - reader.skipField(); - break; - } - } - return msg; + * @param {?proto.ml_metadata.MySQLDatabaseConfig|undefined} value + * @return {!proto.ml_metadata.ConnectionConfig} returns this +*/ +proto.ml_metadata.ConnectionConfig.prototype.setMysql = function(value) { + return jspb.Message.setOneofWrapperField(this, 2, proto.ml_metadata.ConnectionConfig.oneofGroups_[0], value); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ConnectionConfig} returns this */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.ConnectionConfig.prototype.clearMysql = function() { + return this.setMysql(undefined); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( - 1, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeString( - 3, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); - if (f != null) { - writer.writeBool( - 4, - f - ); - } +proto.ml_metadata.ConnectionConfig.prototype.hasMysql = function() { + return jspb.Message.getField(this, 2) != null; }; /** - * optional string server_key = 1; - * @return {string} + * optional SqliteMetadataSourceConfig sqlite = 3; + * @return {?proto.ml_metadata.SqliteMetadataSourceConfig} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getServerKey = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +proto.ml_metadata.ConnectionConfig.prototype.getSqlite = function() { + return /** @type{?proto.ml_metadata.SqliteMetadataSourceConfig} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.SqliteMetadataSourceConfig, 3)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this - */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setServerKey = function(value) { - return jspb.Message.setField(this, 1, value); + * @param {?proto.ml_metadata.SqliteMetadataSourceConfig|undefined} value + * @return {!proto.ml_metadata.ConnectionConfig} returns this +*/ +proto.ml_metadata.ConnectionConfig.prototype.setSqlite = function(value) { + return jspb.Message.setOneofWrapperField(this, 3, proto.ml_metadata.ConnectionConfig.oneofGroups_[0], value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ConnectionConfig} returns this */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearServerKey = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.ConnectionConfig.prototype.clearSqlite = function() { + return this.setSqlite(undefined); }; @@ -10697,35 +11126,36 @@ proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearServerKey = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasServerKey = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.ConnectionConfig.prototype.hasSqlite = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * optional string server_cert = 2; - * @return {string} + * optional PostgreSQLDatabaseConfig postgresql = 5; + * @return {?proto.ml_metadata.PostgreSQLDatabaseConfig} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getServerCert = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.ConnectionConfig.prototype.getPostgresql = function() { + return /** @type{?proto.ml_metadata.PostgreSQLDatabaseConfig} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.PostgreSQLDatabaseConfig, 5)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this - */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setServerCert = function(value) { - return jspb.Message.setField(this, 2, value); + * @param {?proto.ml_metadata.PostgreSQLDatabaseConfig|undefined} value + * @return {!proto.ml_metadata.ConnectionConfig} returns this +*/ +proto.ml_metadata.ConnectionConfig.prototype.setPostgresql = function(value) { + return jspb.Message.setOneofWrapperField(this, 5, proto.ml_metadata.ConnectionConfig.oneofGroups_[0], value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ConnectionConfig} returns this */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearServerCert = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.ConnectionConfig.prototype.clearPostgresql = function() { + return this.setPostgresql(undefined); }; @@ -10733,35 +11163,36 @@ proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearServerCert * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasServerCert = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.ConnectionConfig.prototype.hasPostgresql = function() { + return jspb.Message.getField(this, 5) != null; }; /** - * optional string custom_ca = 3; - * @return {string} + * optional RetryOptions retry_options = 4; + * @return {?proto.ml_metadata.RetryOptions} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getCustomCa = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +proto.ml_metadata.ConnectionConfig.prototype.getRetryOptions = function() { + return /** @type{?proto.ml_metadata.RetryOptions} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.RetryOptions, 4)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this - */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setCustomCa = function(value) { - return jspb.Message.setField(this, 3, value); + * @param {?proto.ml_metadata.RetryOptions|undefined} value + * @return {!proto.ml_metadata.ConnectionConfig} returns this +*/ +proto.ml_metadata.ConnectionConfig.prototype.setRetryOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ConnectionConfig} returns this */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearCustomCa = function() { - return jspb.Message.setField(this, 3, undefined); +proto.ml_metadata.ConnectionConfig.prototype.clearRetryOptions = function() { + return this.setRetryOptions(undefined); }; @@ -10769,109 +11200,159 @@ proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearCustomCa = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasCustomCa = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.ConnectionConfig.prototype.hasRetryOptions = function() { + return jspb.Message.getField(this, 4) != null; }; -/** - * optional bool client_verify = 4; - * @return {boolean} - */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getClientVerify = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, false)); -}; - -/** - * @param {boolean} value - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this - */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setClientVerify = function(value) { - return jspb.Message.setField(this, 4, value); -}; +if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearClientVerify = function() { - return jspb.Message.setField(this, 4, undefined); +proto.ml_metadata.GrpcChannelArguments.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GrpcChannelArguments.toObject(opt_includeInstance, this); }; /** - * Returns whether this field is set. - * @return {boolean} + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.GrpcChannelArguments} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasClientVerify = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GrpcChannelArguments.toObject = function(includeInstance, msg) { + var f, obj = { + maxReceiveMessageLength: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + http2MaxPingStrikes: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; }; +} /** - * optional ConnectionConfig connection_config = 1; - * @return {?proto.ml_metadata.ConnectionConfig} + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.GrpcChannelArguments} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.getConnectionConfig = function() { - return /** @type{?proto.ml_metadata.ConnectionConfig} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.ConnectionConfig, 1)); +proto.ml_metadata.GrpcChannelArguments.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.GrpcChannelArguments; + return proto.ml_metadata.GrpcChannelArguments.deserializeBinaryFromReader(msg, reader); }; /** - * @param {?proto.ml_metadata.ConnectionConfig|undefined} value - * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this -*/ -proto.ml_metadata.MetadataStoreServerConfig.prototype.setConnectionConfig = function(value) { - return jspb.Message.setWrapperField(this, 1, value); + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.GrpcChannelArguments} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.GrpcChannelArguments} + */ +proto.ml_metadata.GrpcChannelArguments.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt64()); + msg.setMaxReceiveMessageLength(value); + break; + case 2: + var value = /** @type {number} */ (reader.readInt64()); + msg.setHttp2MaxPingStrikes(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.clearConnectionConfig = function() { - return this.setConnectionConfig(undefined); +proto.ml_metadata.GrpcChannelArguments.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GrpcChannelArguments.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * Returns whether this field is set. - * @return {boolean} + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GrpcChannelArguments} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.hasConnectionConfig = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GrpcChannelArguments.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( + 1, + f + ); + } + f = /** @type {number} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeInt64( + 2, + f + ); + } }; /** - * optional MigrationOptions migration_options = 3; - * @return {?proto.ml_metadata.MigrationOptions} + * optional int64 max_receive_message_length = 1; + * @return {number} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.getMigrationOptions = function() { - return /** @type{?proto.ml_metadata.MigrationOptions} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.MigrationOptions, 3)); +proto.ml_metadata.GrpcChannelArguments.prototype.getMaxReceiveMessageLength = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** - * @param {?proto.ml_metadata.MigrationOptions|undefined} value - * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this -*/ -proto.ml_metadata.MetadataStoreServerConfig.prototype.setMigrationOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); + * @param {number} value + * @return {!proto.ml_metadata.GrpcChannelArguments} returns this + */ +proto.ml_metadata.GrpcChannelArguments.prototype.setMaxReceiveMessageLength = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GrpcChannelArguments} returns this */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.clearMigrationOptions = function() { - return this.setMigrationOptions(undefined); +proto.ml_metadata.GrpcChannelArguments.prototype.clearMaxReceiveMessageLength = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -10879,36 +11360,35 @@ proto.ml_metadata.MetadataStoreServerConfig.prototype.clearMigrationOptions = fu * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.hasMigrationOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GrpcChannelArguments.prototype.hasMaxReceiveMessageLength = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional SSLConfig ssl_config = 2; - * @return {?proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} + * optional int64 http2_max_ping_strikes = 2; + * @return {number} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.getSslConfig = function() { - return /** @type{?proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.MetadataStoreServerConfig.SSLConfig, 2)); +proto.ml_metadata.GrpcChannelArguments.prototype.getHttp2MaxPingStrikes = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); }; /** - * @param {?proto.ml_metadata.MetadataStoreServerConfig.SSLConfig|undefined} value - * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this -*/ -proto.ml_metadata.MetadataStoreServerConfig.prototype.setSslConfig = function(value) { - return jspb.Message.setWrapperField(this, 2, value); + * @param {number} value + * @return {!proto.ml_metadata.GrpcChannelArguments} returns this + */ +proto.ml_metadata.GrpcChannelArguments.prototype.setHttp2MaxPingStrikes = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GrpcChannelArguments} returns this */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.clearSslConfig = function() { - return this.setSslConfig(undefined); +proto.ml_metadata.GrpcChannelArguments.prototype.clearHttp2MaxPingStrikes = function() { + return jspb.Message.setField(this, 2, undefined); }; @@ -10916,7 +11396,7 @@ proto.ml_metadata.MetadataStoreServerConfig.prototype.clearSslConfig = function( * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.MetadataStoreServerConfig.prototype.hasSslConfig = function() { +proto.ml_metadata.GrpcChannelArguments.prototype.hasHttp2MaxPingStrikes = function() { return jspb.Message.getField(this, 2) != null; }; @@ -10937,8 +11417,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.ListOperationOptions.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ListOperationOptions.toObject(opt_includeInstance, this); +proto.ml_metadata.MetadataStoreClientConfig.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.MetadataStoreClientConfig.toObject(opt_includeInstance, this); }; @@ -10947,16 +11427,17 @@ proto.ml_metadata.ListOperationOptions.prototype.toObject = function(opt_include * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ListOperationOptions} msg The msg instance to transform. + * @param {!proto.ml_metadata.MetadataStoreClientConfig} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ListOperationOptions.toObject = function(includeInstance, msg) { +proto.ml_metadata.MetadataStoreClientConfig.toObject = function(includeInstance, msg) { var f, obj = { - maxResultSize: jspb.Message.getFieldWithDefault(msg, 1, 20), - orderByField: (f = msg.getOrderByField()) && proto.ml_metadata.ListOperationOptions.OrderByField.toObject(includeInstance, f), - nextPageToken: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, - filterQuery: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f + host: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + port: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + sslConfig: (f = msg.getSslConfig()) && proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.toObject(includeInstance, f), + channelArguments: (f = msg.getChannelArguments()) && proto.ml_metadata.GrpcChannelArguments.toObject(includeInstance, f), + clientTimeoutSec: (f = jspb.Message.getOptionalFloatingPointField(msg, 5)) == null ? undefined : f }; if (includeInstance) { @@ -10970,23 +11451,23 @@ proto.ml_metadata.ListOperationOptions.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ListOperationOptions} + * @return {!proto.ml_metadata.MetadataStoreClientConfig} */ -proto.ml_metadata.ListOperationOptions.deserializeBinary = function(bytes) { +proto.ml_metadata.MetadataStoreClientConfig.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ListOperationOptions; - return proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.MetadataStoreClientConfig; + return proto.ml_metadata.MetadataStoreClientConfig.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.ListOperationOptions} msg The message object to deserialize into. + * @param {!proto.ml_metadata.MetadataStoreClientConfig} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ListOperationOptions} + * @return {!proto.ml_metadata.MetadataStoreClientConfig} */ -proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.MetadataStoreClientConfig.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -10994,21 +11475,26 @@ proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader = function(ms var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt32()); - msg.setMaxResultSize(value); + var value = /** @type {string} */ (reader.readString()); + msg.setHost(value); break; case 2: - var value = new proto.ml_metadata.ListOperationOptions.OrderByField; - reader.readMessage(value,proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader); - msg.setOrderByField(value); + var value = /** @type {number} */ (reader.readInt64()); + msg.setPort(value); break; case 3: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); + var value = new proto.ml_metadata.MetadataStoreClientConfig.SSLConfig; + reader.readMessage(value,proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinaryFromReader); + msg.setSslConfig(value); break; case 4: - var value = /** @type {string} */ (reader.readString()); - msg.setFilterQuery(value); + var value = new proto.ml_metadata.GrpcChannelArguments; + reader.readMessage(value,proto.ml_metadata.GrpcChannelArguments.deserializeBinaryFromReader); + msg.setChannelArguments(value); + break; + case 5: + var value = /** @type {number} */ (reader.readDouble()); + msg.setClientTimeoutSec(value); break; default: reader.skipField(); @@ -11023,9 +11509,9 @@ proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader = function(ms * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.ListOperationOptions.prototype.serializeBinary = function() { +proto.ml_metadata.MetadataStoreClientConfig.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter(this, writer); + proto.ml_metadata.MetadataStoreClientConfig.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11033,38 +11519,46 @@ proto.ml_metadata.ListOperationOptions.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ListOperationOptions} message + * @param {!proto.ml_metadata.MetadataStoreClientConfig} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.MetadataStoreClientConfig.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeInt32( + writer.writeString( 1, f ); } - f = message.getOrderByField(); + f = /** @type {number} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeMessage( + writer.writeInt64( 2, - f, - proto.ml_metadata.ListOperationOptions.OrderByField.serializeBinaryToWriter + f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); + f = message.getSslConfig(); if (f != null) { - writer.writeString( + writer.writeMessage( 3, - f + f, + proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.serializeBinaryToWriter ); } - f = /** @type {string} */ (jspb.Message.getField(message, 4)); + f = message.getChannelArguments(); if (f != null) { - writer.writeString( + writer.writeMessage( 4, + f, + proto.ml_metadata.GrpcChannelArguments.serializeBinaryToWriter + ); + } + f = /** @type {number} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeDouble( + 5, f ); } @@ -11087,8 +11581,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ListOperationOptions.OrderByField.toObject(opt_includeInstance, this); +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.toObject(opt_includeInstance, this); }; @@ -11097,14 +11591,15 @@ proto.ml_metadata.ListOperationOptions.OrderByField.prototype.toObject = functio * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ListOperationOptions.OrderByField} msg The msg instance to transform. + * @param {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ListOperationOptions.OrderByField.toObject = function(includeInstance, msg) { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.toObject = function(includeInstance, msg) { var f, obj = { - field: jspb.Message.getFieldWithDefault(msg, 1, 3), - isAsc: jspb.Message.getBooleanFieldWithDefault(msg, 2, true) + clientKey: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + serverCert: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + customCa: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f }; if (includeInstance) { @@ -11118,23 +11613,23 @@ proto.ml_metadata.ListOperationOptions.OrderByField.toObject = function(includeI /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} */ -proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinary = function(bytes) { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ListOperationOptions.OrderByField; - return proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.MetadataStoreClientConfig.SSLConfig; + return proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.ListOperationOptions.OrderByField} msg The message object to deserialize into. + * @param {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} */ -proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11142,12 +11637,16 @@ proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} */ (reader.readEnum()); - msg.setField(value); + var value = /** @type {string} */ (reader.readString()); + msg.setClientKey(value); break; case 2: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setIsAsc(value); + var value = /** @type {string} */ (reader.readString()); + msg.setServerCert(value); + break; + case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setCustomCa(value); break; default: reader.skipField(); @@ -11162,9 +11661,9 @@ proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.serializeBinary = function() { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ListOperationOptions.OrderByField.serializeBinaryToWriter(this, writer); + proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11172,62 +11671,59 @@ proto.ml_metadata.ListOperationOptions.OrderByField.prototype.serializeBinary = /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ListOperationOptions.OrderByField} message + * @param {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ListOperationOptions.OrderByField.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} */ (jspb.Message.getField(message, 1)); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeEnum( + writer.writeString( 1, f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); + f = /** @type {string} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeBool( + writer.writeString( 2, f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } }; /** - * @enum {number} - */ -proto.ml_metadata.ListOperationOptions.OrderByField.Field = { - FIELD_UNSPECIFIED: 0, - CREATE_TIME: 1, - LAST_UPDATE_TIME: 2, - ID: 3 -}; - -/** - * optional Field field = 1; - * @return {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} + * optional string client_key = 1; + * @return {string} */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.getField = function() { - return /** @type {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} */ (jspb.Message.getFieldWithDefault(this, 1, 3)); +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.getClientKey = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} value - * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + * @param {string} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.setField = function(value) { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.setClientKey = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.clearField = function() { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearClientKey = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -11236,34 +11732,34 @@ proto.ml_metadata.ListOperationOptions.OrderByField.prototype.clearField = funct * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.hasField = function() { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.hasClientKey = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional bool is_asc = 2; - * @return {boolean} + * optional string server_cert = 2; + * @return {string} */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.getIsAsc = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, true)); +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.getServerCert = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + * @param {string} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.setIsAsc = function(value) { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.setServerCert = function(value) { return jspb.Message.setField(this, 2, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.clearIsAsc = function() { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearServerCert = function() { return jspb.Message.setField(this, 2, undefined); }; @@ -11272,35 +11768,35 @@ proto.ml_metadata.ListOperationOptions.OrderByField.prototype.clearIsAsc = funct * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationOptions.OrderByField.prototype.hasIsAsc = function() { +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.hasServerCert = function() { return jspb.Message.getField(this, 2) != null; }; /** - * optional int32 max_result_size = 1; - * @return {number} + * optional string custom_ca = 3; + * @return {string} */ -proto.ml_metadata.ListOperationOptions.prototype.getMaxResultSize = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 20)); +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.getCustomCa = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); }; /** - * @param {number} value - * @return {!proto.ml_metadata.ListOperationOptions} returns this + * @param {string} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this */ -proto.ml_metadata.ListOperationOptions.prototype.setMaxResultSize = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.setCustomCa = function(value) { + return jspb.Message.setField(this, 3, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.ListOperationOptions} returns this + * @return {!proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} returns this */ -proto.ml_metadata.ListOperationOptions.prototype.clearMaxResultSize = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.clearCustomCa = function() { + return jspb.Message.setField(this, 3, undefined); }; @@ -11308,36 +11804,35 @@ proto.ml_metadata.ListOperationOptions.prototype.clearMaxResultSize = function() * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationOptions.prototype.hasMaxResultSize = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.MetadataStoreClientConfig.SSLConfig.prototype.hasCustomCa = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * optional OrderByField order_by_field = 2; - * @return {?proto.ml_metadata.ListOperationOptions.OrderByField} + * optional string host = 1; + * @return {string} */ -proto.ml_metadata.ListOperationOptions.prototype.getOrderByField = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions.OrderByField} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.ListOperationOptions.OrderByField, 2)); +proto.ml_metadata.MetadataStoreClientConfig.prototype.getHost = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {?proto.ml_metadata.ListOperationOptions.OrderByField|undefined} value - * @return {!proto.ml_metadata.ListOperationOptions} returns this -*/ -proto.ml_metadata.ListOperationOptions.prototype.setOrderByField = function(value) { - return jspb.Message.setWrapperField(this, 2, value); + * @param {string} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + */ +proto.ml_metadata.MetadataStoreClientConfig.prototype.setHost = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ListOperationOptions} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this */ -proto.ml_metadata.ListOperationOptions.prototype.clearOrderByField = function() { - return this.setOrderByField(undefined); +proto.ml_metadata.MetadataStoreClientConfig.prototype.clearHost = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -11345,35 +11840,35 @@ proto.ml_metadata.ListOperationOptions.prototype.clearOrderByField = function() * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationOptions.prototype.hasOrderByField = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.MetadataStoreClientConfig.prototype.hasHost = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional string next_page_token = 3; - * @return {string} + * optional int64 port = 2; + * @return {number} */ -proto.ml_metadata.ListOperationOptions.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +proto.ml_metadata.MetadataStoreClientConfig.prototype.getPort = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.ListOperationOptions} returns this + * @param {number} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this */ -proto.ml_metadata.ListOperationOptions.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 3, value); +proto.ml_metadata.MetadataStoreClientConfig.prototype.setPort = function(value) { + return jspb.Message.setField(this, 2, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.ListOperationOptions} returns this + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this */ -proto.ml_metadata.ListOperationOptions.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 3, undefined); +proto.ml_metadata.MetadataStoreClientConfig.prototype.clearPort = function() { + return jspb.Message.setField(this, 2, undefined); }; @@ -11381,35 +11876,109 @@ proto.ml_metadata.ListOperationOptions.prototype.clearNextPageToken = function() * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationOptions.prototype.hasNextPageToken = function() { +proto.ml_metadata.MetadataStoreClientConfig.prototype.hasPort = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional SSLConfig ssl_config = 3; + * @return {?proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} + */ +proto.ml_metadata.MetadataStoreClientConfig.prototype.getSslConfig = function() { + return /** @type{?proto.ml_metadata.MetadataStoreClientConfig.SSLConfig} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.MetadataStoreClientConfig.SSLConfig, 3)); +}; + + +/** + * @param {?proto.ml_metadata.MetadataStoreClientConfig.SSLConfig|undefined} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this +*/ +proto.ml_metadata.MetadataStoreClientConfig.prototype.setSslConfig = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + */ +proto.ml_metadata.MetadataStoreClientConfig.prototype.clearSslConfig = function() { + return this.setSslConfig(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreClientConfig.prototype.hasSslConfig = function() { return jspb.Message.getField(this, 3) != null; }; /** - * optional string filter_query = 4; - * @return {string} + * optional GrpcChannelArguments channel_arguments = 4; + * @return {?proto.ml_metadata.GrpcChannelArguments} */ -proto.ml_metadata.ListOperationOptions.prototype.getFilterQuery = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, "")); +proto.ml_metadata.MetadataStoreClientConfig.prototype.getChannelArguments = function() { + return /** @type{?proto.ml_metadata.GrpcChannelArguments} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.GrpcChannelArguments, 4)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.ListOperationOptions} returns this + * @param {?proto.ml_metadata.GrpcChannelArguments|undefined} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this +*/ +proto.ml_metadata.MetadataStoreClientConfig.prototype.setChannelArguments = function(value) { + return jspb.Message.setWrapperField(this, 4, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this */ -proto.ml_metadata.ListOperationOptions.prototype.setFilterQuery = function(value) { - return jspb.Message.setField(this, 4, value); +proto.ml_metadata.MetadataStoreClientConfig.prototype.clearChannelArguments = function() { + return this.setChannelArguments(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreClientConfig.prototype.hasChannelArguments = function() { + return jspb.Message.getField(this, 4) != null; +}; + + +/** + * optional double client_timeout_sec = 5; + * @return {number} + */ +proto.ml_metadata.MetadataStoreClientConfig.prototype.getClientTimeoutSec = function() { + return /** @type {number} */ (jspb.Message.getFloatingPointFieldWithDefault(this, 5, 0.0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this + */ +proto.ml_metadata.MetadataStoreClientConfig.prototype.setClientTimeoutSec = function(value) { + return jspb.Message.setField(this, 5, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.ListOperationOptions} returns this + * @return {!proto.ml_metadata.MetadataStoreClientConfig} returns this */ -proto.ml_metadata.ListOperationOptions.prototype.clearFilterQuery = function() { - return jspb.Message.setField(this, 4, undefined); +proto.ml_metadata.MetadataStoreClientConfig.prototype.clearClientTimeoutSec = function() { + return jspb.Message.setField(this, 5, undefined); }; @@ -11417,18 +11986,1683 @@ proto.ml_metadata.ListOperationOptions.prototype.clearFilterQuery = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationOptions.prototype.hasFilterQuery = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.MetadataStoreClientConfig.prototype.hasClientTimeoutSec = function() { + return jspb.Message.getField(this, 5) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.MetadataStoreServerConfig.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.MetadataStoreServerConfig} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.MetadataStoreServerConfig.toObject = function(includeInstance, msg) { + var f, obj = { + connectionConfig: (f = msg.getConnectionConfig()) && proto.ml_metadata.ConnectionConfig.toObject(includeInstance, f), + migrationOptions: (f = msg.getMigrationOptions()) && proto.ml_metadata.MigrationOptions.toObject(includeInstance, f), + sslConfig: (f = msg.getSslConfig()) && proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.MetadataStoreServerConfig} + */ +proto.ml_metadata.MetadataStoreServerConfig.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.MetadataStoreServerConfig; + return proto.ml_metadata.MetadataStoreServerConfig.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.MetadataStoreServerConfig} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.MetadataStoreServerConfig} + */ +proto.ml_metadata.MetadataStoreServerConfig.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.ml_metadata.ConnectionConfig; + reader.readMessage(value,proto.ml_metadata.ConnectionConfig.deserializeBinaryFromReader); + msg.setConnectionConfig(value); + break; + case 3: + var value = new proto.ml_metadata.MigrationOptions; + reader.readMessage(value,proto.ml_metadata.MigrationOptions.deserializeBinaryFromReader); + msg.setMigrationOptions(value); + break; + case 2: + var value = new proto.ml_metadata.MetadataStoreServerConfig.SSLConfig; + reader.readMessage(value,proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinaryFromReader); + msg.setSslConfig(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.MetadataStoreServerConfig.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.MetadataStoreServerConfig} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.MetadataStoreServerConfig.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getConnectionConfig(); + if (f != null) { + writer.writeMessage( + 1, + f, + proto.ml_metadata.ConnectionConfig.serializeBinaryToWriter + ); + } + f = message.getMigrationOptions(); + if (f != null) { + writer.writeMessage( + 3, + f, + proto.ml_metadata.MigrationOptions.serializeBinaryToWriter + ); + } + f = message.getSslConfig(); + if (f != null) { + writer.writeMessage( + 2, + f, + proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.serializeBinaryToWriter + ); + } +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.toObject = function(includeInstance, msg) { + var f, obj = { + serverKey: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + serverCert: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + customCa: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + clientVerify: (f = jspb.Message.getBooleanField(msg, 4)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.MetadataStoreServerConfig.SSLConfig; + return proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setServerKey(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setServerCert(value); + break; + case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setCustomCa(value); + break; + case 4: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setClientVerify(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( + 1, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeBool( + 4, + f + ); + } +}; + + +/** + * optional string server_key = 1; + * @return {string} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getServerKey = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setServerKey = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearServerKey = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasServerKey = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional string server_cert = 2; + * @return {string} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getServerCert = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setServerCert = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearServerCert = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasServerCert = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional string custom_ca = 3; + * @return {string} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getCustomCa = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setCustomCa = function(value) { + return jspb.Message.setField(this, 3, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearCustomCa = function() { + return jspb.Message.setField(this, 3, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasCustomCa = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional bool client_verify = 4; + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.getClientVerify = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.setClientVerify = function(value) { + return jspb.Message.setField(this, 4, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.clearClientVerify = function() { + return jspb.Message.setField(this, 4, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.SSLConfig.prototype.hasClientVerify = function() { + return jspb.Message.getField(this, 4) != null; +}; + + +/** + * optional ConnectionConfig connection_config = 1; + * @return {?proto.ml_metadata.ConnectionConfig} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.getConnectionConfig = function() { + return /** @type{?proto.ml_metadata.ConnectionConfig} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.ConnectionConfig, 1)); +}; + + +/** + * @param {?proto.ml_metadata.ConnectionConfig|undefined} value + * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this +*/ +proto.ml_metadata.MetadataStoreServerConfig.prototype.setConnectionConfig = function(value) { + return jspb.Message.setWrapperField(this, 1, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.clearConnectionConfig = function() { + return this.setConnectionConfig(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.hasConnectionConfig = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional MigrationOptions migration_options = 3; + * @return {?proto.ml_metadata.MigrationOptions} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.getMigrationOptions = function() { + return /** @type{?proto.ml_metadata.MigrationOptions} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.MigrationOptions, 3)); +}; + + +/** + * @param {?proto.ml_metadata.MigrationOptions|undefined} value + * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this +*/ +proto.ml_metadata.MetadataStoreServerConfig.prototype.setMigrationOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.clearMigrationOptions = function() { + return this.setMigrationOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.hasMigrationOptions = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional SSLConfig ssl_config = 2; + * @return {?proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.getSslConfig = function() { + return /** @type{?proto.ml_metadata.MetadataStoreServerConfig.SSLConfig} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.MetadataStoreServerConfig.SSLConfig, 2)); +}; + + +/** + * @param {?proto.ml_metadata.MetadataStoreServerConfig.SSLConfig|undefined} value + * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this +*/ +proto.ml_metadata.MetadataStoreServerConfig.prototype.setSslConfig = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.MetadataStoreServerConfig} returns this + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.clearSslConfig = function() { + return this.setSslConfig(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.MetadataStoreServerConfig.prototype.hasSslConfig = function() { + return jspb.Message.getField(this, 2) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.ListOperationOptions.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ListOperationOptions.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.ListOperationOptions} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ListOperationOptions.toObject = function(includeInstance, msg) { + var f, obj = { + maxResultSize: jspb.Message.getFieldWithDefault(msg, 1, 20), + orderByField: (f = msg.getOrderByField()) && proto.ml_metadata.ListOperationOptions.OrderByField.toObject(includeInstance, f), + nextPageToken: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + filterQuery: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.ListOperationOptions} + */ +proto.ml_metadata.ListOperationOptions.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.ListOperationOptions; + return proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.ListOperationOptions} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.ListOperationOptions} + */ +proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt32()); + msg.setMaxResultSize(value); + break; + case 2: + var value = new proto.ml_metadata.ListOperationOptions.OrderByField; + reader.readMessage(value,proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader); + msg.setOrderByField(value); + break; + case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setNextPageToken(value); + break; + case 4: + var value = /** @type {string} */ (reader.readString()); + msg.setFilterQuery(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.ListOperationOptions.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.ListOperationOptions} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt32( + 1, + f + ); + } + f = message.getOrderByField(); + if (f != null) { + writer.writeMessage( + 2, + f, + proto.ml_metadata.ListOperationOptions.OrderByField.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeString( + 4, + f + ); + } +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ListOperationOptions.OrderByField.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.ListOperationOptions.OrderByField} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ListOperationOptions.OrderByField.toObject = function(includeInstance, msg) { + var f, obj = { + field: jspb.Message.getFieldWithDefault(msg, 1, 3), + isAsc: jspb.Message.getBooleanFieldWithDefault(msg, 2, true) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.ListOperationOptions.OrderByField; + return proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.ListOperationOptions.OrderByField} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} */ (reader.readEnum()); + msg.setField(value); + break; + case 2: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setIsAsc(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.ListOperationOptions.OrderByField.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.ListOperationOptions.OrderByField} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ListOperationOptions.OrderByField.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeEnum( + 1, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeBool( + 2, + f + ); + } +}; + + +/** + * @enum {number} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.Field = { + FIELD_UNSPECIFIED: 0, + CREATE_TIME: 1, + LAST_UPDATE_TIME: 2, + ID: 3 +}; + +/** + * optional Field field = 1; + * @return {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.getField = function() { + return /** @type {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} */ (jspb.Message.getFieldWithDefault(this, 1, 3)); +}; + + +/** + * @param {!proto.ml_metadata.ListOperationOptions.OrderByField.Field} value + * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.setField = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.clearField = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.hasField = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional bool is_asc = 2; + * @return {boolean} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.getIsAsc = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, true)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.setIsAsc = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ListOperationOptions.OrderByField} returns this + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.clearIsAsc = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationOptions.OrderByField.prototype.hasIsAsc = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional int32 max_result_size = 1; + * @return {number} + */ +proto.ml_metadata.ListOperationOptions.prototype.getMaxResultSize = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 20)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.ListOperationOptions} returns this + */ +proto.ml_metadata.ListOperationOptions.prototype.setMaxResultSize = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ListOperationOptions} returns this + */ +proto.ml_metadata.ListOperationOptions.prototype.clearMaxResultSize = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationOptions.prototype.hasMaxResultSize = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional OrderByField order_by_field = 2; + * @return {?proto.ml_metadata.ListOperationOptions.OrderByField} + */ +proto.ml_metadata.ListOperationOptions.prototype.getOrderByField = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions.OrderByField} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.ListOperationOptions.OrderByField, 2)); +}; + + +/** + * @param {?proto.ml_metadata.ListOperationOptions.OrderByField|undefined} value + * @return {!proto.ml_metadata.ListOperationOptions} returns this +*/ +proto.ml_metadata.ListOperationOptions.prototype.setOrderByField = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ListOperationOptions} returns this + */ +proto.ml_metadata.ListOperationOptions.prototype.clearOrderByField = function() { + return this.setOrderByField(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationOptions.prototype.hasOrderByField = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional string next_page_token = 3; + * @return {string} + */ +proto.ml_metadata.ListOperationOptions.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.ListOperationOptions} returns this + */ +proto.ml_metadata.ListOperationOptions.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 3, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ListOperationOptions} returns this + */ +proto.ml_metadata.ListOperationOptions.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 3, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationOptions.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional string filter_query = 4; + * @return {string} + */ +proto.ml_metadata.ListOperationOptions.prototype.getFilterQuery = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.ListOperationOptions} returns this + */ +proto.ml_metadata.ListOperationOptions.prototype.setFilterQuery = function(value) { + return jspb.Message.setField(this, 4, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ListOperationOptions} returns this + */ +proto.ml_metadata.ListOperationOptions.prototype.clearFilterQuery = function() { + return jspb.Message.setField(this, 4, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationOptions.prototype.hasFilterQuery = function() { + return jspb.Message.getField(this, 4) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.ListOperationNextPageToken.repeatedFields_ = [4]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ListOperationNextPageToken.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.ListOperationNextPageToken} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ListOperationNextPageToken.toObject = function(includeInstance, msg) { + var f, obj = { + idOffset: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + fieldOffset: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + setOptions: (f = msg.getSetOptions()) && proto.ml_metadata.ListOperationOptions.toObject(includeInstance, f), + listedIdsList: (f = jspb.Message.getRepeatedField(msg, 4)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.ListOperationNextPageToken} + */ +proto.ml_metadata.ListOperationNextPageToken.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.ListOperationNextPageToken; + return proto.ml_metadata.ListOperationNextPageToken.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.ListOperationNextPageToken} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.ListOperationNextPageToken} + */ +proto.ml_metadata.ListOperationNextPageToken.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt64()); + msg.setIdOffset(value); + break; + case 2: + var value = /** @type {number} */ (reader.readInt64()); + msg.setFieldOffset(value); + break; + case 3: + var value = new proto.ml_metadata.ListOperationOptions; + reader.readMessage(value,proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader); + msg.setSetOptions(value); + break; + case 4: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addListedIds(values[i]); + } + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.ListOperationNextPageToken.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.ListOperationNextPageToken} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ListOperationNextPageToken.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( + 1, + f + ); + } + f = /** @type {number} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeInt64( + 2, + f + ); + } + f = message.getSetOptions(); + if (f != null) { + writer.writeMessage( + 3, + f, + proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter + ); + } + f = message.getListedIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 4, + f + ); + } +}; + + +/** + * optional int64 id_offset = 1; + * @return {number} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.getIdOffset = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.setIdOffset = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.clearIdOffset = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.hasIdOffset = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional int64 field_offset = 2; + * @return {number} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.getFieldOffset = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.setFieldOffset = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.clearFieldOffset = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.hasFieldOffset = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional ListOperationOptions set_options = 3; + * @return {?proto.ml_metadata.ListOperationOptions} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.getSetOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.ListOperationOptions, 3)); +}; + + +/** + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this +*/ +proto.ml_metadata.ListOperationNextPageToken.prototype.setSetOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.clearSetOptions = function() { + return this.setSetOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.hasSetOptions = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * repeated int64 listed_ids = 4; + * @return {!Array} + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.getListedIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 4)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.setListedIdsList = function(value) { + return jspb.Message.setField(this, 4, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.addListedIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 4, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + */ +proto.ml_metadata.ListOperationNextPageToken.prototype.clearListedIdsList = function() { + return this.setListedIdsList([]); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.TransactionOptions.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.TransactionOptions.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.TransactionOptions} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.TransactionOptions.toObject = function(includeInstance, msg) { + var f, obj = { + tag: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + }; + + jspb.Message.toObjectExtension(/** @type {!jspb.Message} */ (msg), obj, + proto.ml_metadata.TransactionOptions.extensions, proto.ml_metadata.TransactionOptions.prototype.getExtension, + includeInstance); + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.TransactionOptions.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.TransactionOptions; + return proto.ml_metadata.TransactionOptions.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.TransactionOptions} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.TransactionOptions.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setTag(value); + break; + default: + jspb.Message.readBinaryExtension(msg, reader, + proto.ml_metadata.TransactionOptions.extensionsBinary, + proto.ml_metadata.TransactionOptions.prototype.getExtension, + proto.ml_metadata.TransactionOptions.prototype.setExtension); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.TransactionOptions.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.TransactionOptions.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.TransactionOptions} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.TransactionOptions.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( + 1, + f + ); + } + jspb.Message.serializeBinaryExtensions(message, writer, + proto.ml_metadata.TransactionOptions.extensionsBinary, proto.ml_metadata.TransactionOptions.prototype.getExtension); +}; + + +/** + * optional string tag = 1; + * @return {string} + */ +proto.ml_metadata.TransactionOptions.prototype.getTag = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.TransactionOptions} returns this + */ +proto.ml_metadata.TransactionOptions.prototype.setTag = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.TransactionOptions} returns this + */ +proto.ml_metadata.TransactionOptions.prototype.clearTag = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.TransactionOptions.prototype.hasTag = function() { + return jspb.Message.getField(this, 1) != null; +}; + + + +/** + * Oneof group definitions for this message. Each group defines the field + * numbers belonging to that group. When of these fields' value is set, all + * other fields in the group are cleared. During deserialization, if multiple + * fields are encountered for a group, only the last value seen will be kept. + * @private {!Array>} + * @const + */ +proto.ml_metadata.LineageGraphQueryOptions.oneofGroups_ = [[1]]; + +/** + * @enum {number} + */ +proto.ml_metadata.LineageGraphQueryOptions.QueryNodesCase = { + QUERY_NODES_NOT_SET: 0, + ARTIFACTS_OPTIONS: 1 +}; + +/** + * @return {proto.ml_metadata.LineageGraphQueryOptions.QueryNodesCase} + */ +proto.ml_metadata.LineageGraphQueryOptions.prototype.getQueryNodesCase = function() { + return /** @type {proto.ml_metadata.LineageGraphQueryOptions.QueryNodesCase} */(jspb.Message.computeOneofCase(this, proto.ml_metadata.LineageGraphQueryOptions.oneofGroups_[0])); +}; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.LineageGraphQueryOptions.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.LineageGraphQueryOptions.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.LineageGraphQueryOptions} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.LineageGraphQueryOptions.toObject = function(includeInstance, msg) { + var f, obj = { + artifactsOptions: (f = msg.getArtifactsOptions()) && proto.ml_metadata.ListOperationOptions.toObject(includeInstance, f), + stopConditions: (f = msg.getStopConditions()) && proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.toObject(includeInstance, f), + maxNodeSize: jspb.Message.getFieldWithDefault(msg, 3, 20) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.LineageGraphQueryOptions} + */ +proto.ml_metadata.LineageGraphQueryOptions.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.LineageGraphQueryOptions; + return proto.ml_metadata.LineageGraphQueryOptions.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.LineageGraphQueryOptions} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.LineageGraphQueryOptions} + */ +proto.ml_metadata.LineageGraphQueryOptions.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.ml_metadata.ListOperationOptions; + reader.readMessage(value,proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader); + msg.setArtifactsOptions(value); + break; + case 2: + var value = new proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint; + reader.readMessage(value,proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryFromReader); + msg.setStopConditions(value); + break; + case 3: + var value = /** @type {number} */ (reader.readInt64()); + msg.setMaxNodeSize(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.LineageGraphQueryOptions.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.LineageGraphQueryOptions.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + /** - * List of repeated fields within this message type. - * @private {!Array} - * @const + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.LineageGraphQueryOptions} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ListOperationNextPageToken.repeatedFields_ = [4]; +proto.ml_metadata.LineageGraphQueryOptions.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactsOptions(); + if (f != null) { + writer.writeMessage( + 1, + f, + proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter + ); + } + f = message.getStopConditions(); + if (f != null) { + writer.writeMessage( + 2, + f, + proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.serializeBinaryToWriter + ); + } + f = /** @type {number} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeInt64( + 3, + f + ); + } +}; + + @@ -11445,8 +13679,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ListOperationNextPageToken.toObject(opt_includeInstance, this); +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.toObject(opt_includeInstance, this); }; @@ -11455,16 +13689,15 @@ proto.ml_metadata.ListOperationNextPageToken.prototype.toObject = function(opt_i * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ListOperationNextPageToken} msg The msg instance to transform. + * @param {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ListOperationNextPageToken.toObject = function(includeInstance, msg) { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.toObject = function(includeInstance, msg) { var f, obj = { - idOffset: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - fieldOffset: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - setOptions: (f = msg.getSetOptions()) && proto.ml_metadata.ListOperationOptions.toObject(includeInstance, f), - listedIdsList: (f = jspb.Message.getRepeatedField(msg, 4)) == null ? undefined : f + maxNumHops: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + boundaryArtifacts: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + boundaryExecutions: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f }; if (includeInstance) { @@ -11478,23 +13711,23 @@ proto.ml_metadata.ListOperationNextPageToken.toObject = function(includeInstance /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ListOperationNextPageToken} + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} */ -proto.ml_metadata.ListOperationNextPageToken.deserializeBinary = function(bytes) { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ListOperationNextPageToken; - return proto.ml_metadata.ListOperationNextPageToken.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint; + return proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.ListOperationNextPageToken} msg The message object to deserialize into. + * @param {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ListOperationNextPageToken} + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} */ -proto.ml_metadata.ListOperationNextPageToken.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11503,22 +13736,15 @@ proto.ml_metadata.ListOperationNextPageToken.deserializeBinaryFromReader = funct switch (field) { case 1: var value = /** @type {number} */ (reader.readInt64()); - msg.setIdOffset(value); + msg.setMaxNumHops(value); break; case 2: - var value = /** @type {number} */ (reader.readInt64()); - msg.setFieldOffset(value); + var value = /** @type {string} */ (reader.readString()); + msg.setBoundaryArtifacts(value); break; case 3: - var value = new proto.ml_metadata.ListOperationOptions; - reader.readMessage(value,proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader); - msg.setSetOptions(value); - break; - case 4: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addListedIds(values[i]); - } + var value = /** @type {string} */ (reader.readString()); + msg.setBoundaryExecutions(value); break; default: reader.skipField(); @@ -11533,9 +13759,9 @@ proto.ml_metadata.ListOperationNextPageToken.deserializeBinaryFromReader = funct * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.serializeBinary = function() { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ListOperationNextPageToken.serializeBinaryToWriter(this, writer); + proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11543,11 +13769,11 @@ proto.ml_metadata.ListOperationNextPageToken.prototype.serializeBinary = functio /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ListOperationNextPageToken} message + * @param {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ListOperationNextPageToken.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.serializeBinaryToWriter = function(message, writer) { var f = undefined; f = /** @type {number} */ (jspb.Message.getField(message, 1)); if (f != null) { @@ -11556,25 +13782,17 @@ proto.ml_metadata.ListOperationNextPageToken.serializeBinaryToWriter = function( f ); } - f = /** @type {number} */ (jspb.Message.getField(message, 2)); + f = /** @type {string} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeInt64( + writer.writeString( 2, f ); } - f = message.getSetOptions(); + f = /** @type {string} */ (jspb.Message.getField(message, 3)); if (f != null) { - writer.writeMessage( + writer.writeString( 3, - f, - proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter - ); - } - f = message.getListedIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( - 4, f ); } @@ -11582,28 +13800,28 @@ proto.ml_metadata.ListOperationNextPageToken.serializeBinaryToWriter = function( /** - * optional int64 id_offset = 1; + * optional int64 max_num_hops = 1; * @return {number} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.getIdOffset = function() { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.getMaxNumHops = function() { return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** * @param {number} value - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this */ -proto.ml_metadata.ListOperationNextPageToken.prototype.setIdOffset = function(value) { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.setMaxNumHops = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this */ -proto.ml_metadata.ListOperationNextPageToken.prototype.clearIdOffset = function() { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearMaxNumHops = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -11612,34 +13830,34 @@ proto.ml_metadata.ListOperationNextPageToken.prototype.clearIdOffset = function( * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.hasIdOffset = function() { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.hasMaxNumHops = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional int64 field_offset = 2; - * @return {number} + * optional string boundary_artifacts = 2; + * @return {string} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.getFieldOffset = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.getBoundaryArtifacts = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {number} value - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + * @param {string} value + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this */ -proto.ml_metadata.ListOperationNextPageToken.prototype.setFieldOffset = function(value) { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.setBoundaryArtifacts = function(value) { return jspb.Message.setField(this, 2, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this */ -proto.ml_metadata.ListOperationNextPageToken.prototype.clearFieldOffset = function() { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearBoundaryArtifacts = function() { return jspb.Message.setField(this, 2, undefined); }; @@ -11648,36 +13866,35 @@ proto.ml_metadata.ListOperationNextPageToken.prototype.clearFieldOffset = functi * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.hasFieldOffset = function() { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.hasBoundaryArtifacts = function() { return jspb.Message.getField(this, 2) != null; }; /** - * optional ListOperationOptions set_options = 3; - * @return {?proto.ml_metadata.ListOperationOptions} + * optional string boundary_executions = 3; + * @return {string} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.getSetOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.ListOperationOptions, 3)); +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.getBoundaryExecutions = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); }; /** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this -*/ -proto.ml_metadata.ListOperationNextPageToken.prototype.setSetOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); + * @param {string} value + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this + */ +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.setBoundaryExecutions = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this */ -proto.ml_metadata.ListOperationNextPageToken.prototype.clearSetOptions = function() { - return this.setSetOptions(undefined); +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearBoundaryExecutions = function() { + return jspb.Message.setField(this, 3, undefined); }; @@ -11685,154 +13902,118 @@ proto.ml_metadata.ListOperationNextPageToken.prototype.clearSetOptions = functio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.hasSetOptions = function() { +proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.hasBoundaryExecutions = function() { return jspb.Message.getField(this, 3) != null; }; /** - * repeated int64 listed_ids = 4; - * @return {!Array} + * optional ListOperationOptions artifacts_options = 1; + * @return {?proto.ml_metadata.ListOperationOptions} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.getListedIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 4)); +proto.ml_metadata.LineageGraphQueryOptions.prototype.getArtifactsOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.ListOperationOptions, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this - */ -proto.ml_metadata.ListOperationNextPageToken.prototype.setListedIdsList = function(value) { - return jspb.Message.setField(this, 4, value || []); + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this +*/ +proto.ml_metadata.LineageGraphQueryOptions.prototype.setArtifactsOptions = function(value) { + return jspb.Message.setOneofWrapperField(this, 1, proto.ml_metadata.LineageGraphQueryOptions.oneofGroups_[0], value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this */ -proto.ml_metadata.ListOperationNextPageToken.prototype.addListedIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 4, value, opt_index); +proto.ml_metadata.LineageGraphQueryOptions.prototype.clearArtifactsOptions = function() { + return this.setArtifactsOptions(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.ListOperationNextPageToken} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.ListOperationNextPageToken.prototype.clearListedIdsList = function() { - return this.setListedIdsList([]); +proto.ml_metadata.LineageGraphQueryOptions.prototype.hasArtifactsOptions = function() { + return jspb.Message.getField(this, 1) != null; }; +/** + * optional BoundaryConstraint stop_conditions = 2; + * @return {?proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} + */ +proto.ml_metadata.LineageGraphQueryOptions.prototype.getStopConditions = function() { + return /** @type{?proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint, 2)); +}; - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} - */ -proto.ml_metadata.TransactionOptions.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.TransactionOptions.toObject(opt_includeInstance, this); + * @param {?proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint|undefined} value + * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this +*/ +proto.ml_metadata.LineageGraphQueryOptions.prototype.setStopConditions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.TransactionOptions} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this */ -proto.ml_metadata.TransactionOptions.toObject = function(includeInstance, msg) { - var f, obj = { - - }; +proto.ml_metadata.LineageGraphQueryOptions.prototype.clearStopConditions = function() { + return this.setStopConditions(undefined); +}; - jspb.Message.toObjectExtension(/** @type {!jspb.Message} */ (msg), obj, - proto.ml_metadata.TransactionOptions.extensions, proto.ml_metadata.TransactionOptions.prototype.getExtension, - includeInstance); - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.LineageGraphQueryOptions.prototype.hasStopConditions = function() { + return jspb.Message.getField(this, 2) != null; }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.TransactionOptions} + * optional int64 max_node_size = 3; + * @return {number} */ -proto.ml_metadata.TransactionOptions.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.TransactionOptions; - return proto.ml_metadata.TransactionOptions.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.LineageGraphQueryOptions.prototype.getMaxNodeSize = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 3, 20)); }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.TransactionOptions} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.TransactionOptions} + * @param {number} value + * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this */ -proto.ml_metadata.TransactionOptions.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - default: - jspb.Message.readBinaryExtension(msg, reader, - proto.ml_metadata.TransactionOptions.extensionsBinary, - proto.ml_metadata.TransactionOptions.prototype.getExtension, - proto.ml_metadata.TransactionOptions.prototype.setExtension); - break; - } - } - return msg; +proto.ml_metadata.LineageGraphQueryOptions.prototype.setMaxNodeSize = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Clears the field making it undefined. + * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this */ -proto.ml_metadata.TransactionOptions.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.TransactionOptions.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.LineageGraphQueryOptions.prototype.clearMaxNodeSize = function() { + return jspb.Message.setField(this, 3, undefined); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.TransactionOptions} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.TransactionOptions.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - jspb.Message.serializeBinaryExtensions(message, writer, - proto.ml_metadata.TransactionOptions.extensionsBinary, proto.ml_metadata.TransactionOptions.prototype.getExtension); +proto.ml_metadata.LineageGraphQueryOptions.prototype.hasMaxNodeSize = function() { + return jspb.Message.getField(this, 3) != null; }; @@ -11845,21 +14026,22 @@ proto.ml_metadata.TransactionOptions.serializeBinaryToWriter = function(message, * @private {!Array>} * @const */ -proto.ml_metadata.LineageGraphQueryOptions.oneofGroups_ = [[1]]; +proto.ml_metadata.LineageSubgraphQueryOptions.oneofGroups_ = [[1,2]]; /** * @enum {number} */ -proto.ml_metadata.LineageGraphQueryOptions.QueryNodesCase = { - QUERY_NODES_NOT_SET: 0, - ARTIFACTS_OPTIONS: 1 +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodesCase = { + STARTING_NODES_NOT_SET: 0, + STARTING_ARTIFACTS: 1, + STARTING_EXECUTIONS: 2 }; /** - * @return {proto.ml_metadata.LineageGraphQueryOptions.QueryNodesCase} + * @return {proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodesCase} */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.getQueryNodesCase = function() { - return /** @type {proto.ml_metadata.LineageGraphQueryOptions.QueryNodesCase} */(jspb.Message.computeOneofCase(this, proto.ml_metadata.LineageGraphQueryOptions.oneofGroups_[0])); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.getStartingNodesCase = function() { + return /** @type {proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodesCase} */(jspb.Message.computeOneofCase(this, proto.ml_metadata.LineageSubgraphQueryOptions.oneofGroups_[0])); }; @@ -11877,8 +14059,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.LineageGraphQueryOptions.toObject(opt_includeInstance, this); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.LineageSubgraphQueryOptions.toObject(opt_includeInstance, this); }; @@ -11887,14 +14069,16 @@ proto.ml_metadata.LineageGraphQueryOptions.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.LineageGraphQueryOptions} msg The msg instance to transform. + * @param {!proto.ml_metadata.LineageSubgraphQueryOptions} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.LineageGraphQueryOptions.toObject = function(includeInstance, msg) { +proto.ml_metadata.LineageSubgraphQueryOptions.toObject = function(includeInstance, msg) { var f, obj = { - artifactsOptions: (f = msg.getArtifactsOptions()) && proto.ml_metadata.ListOperationOptions.toObject(includeInstance, f), - stopConditions: (f = msg.getStopConditions()) && proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.toObject(includeInstance, f) + startingArtifacts: (f = msg.getStartingArtifacts()) && proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.toObject(includeInstance, f), + startingExecutions: (f = msg.getStartingExecutions()) && proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.toObject(includeInstance, f), + maxNumHops: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + direction: (f = jspb.Message.getField(msg, 4)) == null ? undefined : f }; if (includeInstance) { @@ -11908,23 +14092,23 @@ proto.ml_metadata.LineageGraphQueryOptions.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.LineageGraphQueryOptions} + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} */ -proto.ml_metadata.LineageGraphQueryOptions.deserializeBinary = function(bytes) { +proto.ml_metadata.LineageSubgraphQueryOptions.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.LineageGraphQueryOptions; - return proto.ml_metadata.LineageGraphQueryOptions.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.LineageSubgraphQueryOptions; + return proto.ml_metadata.LineageSubgraphQueryOptions.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.LineageGraphQueryOptions} msg The message object to deserialize into. + * @param {!proto.ml_metadata.LineageSubgraphQueryOptions} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.LineageGraphQueryOptions} + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} */ -proto.ml_metadata.LineageGraphQueryOptions.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.LineageSubgraphQueryOptions.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11932,14 +14116,22 @@ proto.ml_metadata.LineageGraphQueryOptions.deserializeBinaryFromReader = functio var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new proto.ml_metadata.ListOperationOptions; - reader.readMessage(value,proto.ml_metadata.ListOperationOptions.deserializeBinaryFromReader); - msg.setArtifactsOptions(value); + var value = new proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes; + reader.readMessage(value,proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.deserializeBinaryFromReader); + msg.setStartingArtifacts(value); break; case 2: - var value = new proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint; - reader.readMessage(value,proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryFromReader); - msg.setStopConditions(value); + var value = new proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes; + reader.readMessage(value,proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.deserializeBinaryFromReader); + msg.setStartingExecutions(value); + break; + case 3: + var value = /** @type {number} */ (reader.readInt64()); + msg.setMaxNumHops(value); + break; + case 4: + var value = /** @type {!proto.ml_metadata.LineageSubgraphQueryOptions.Direction} */ (reader.readEnum()); + msg.setDirection(value); break; default: reader.skipField(); @@ -11954,9 +14146,9 @@ proto.ml_metadata.LineageGraphQueryOptions.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.serializeBinary = function() { +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.LineageGraphQueryOptions.serializeBinaryToWriter(this, writer); + proto.ml_metadata.LineageSubgraphQueryOptions.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11964,31 +14156,55 @@ proto.ml_metadata.LineageGraphQueryOptions.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.LineageGraphQueryOptions} message + * @param {!proto.ml_metadata.LineageSubgraphQueryOptions} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.LineageGraphQueryOptions.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.LineageSubgraphQueryOptions.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactsOptions(); + f = message.getStartingArtifacts(); if (f != null) { writer.writeMessage( 1, f, - proto.ml_metadata.ListOperationOptions.serializeBinaryToWriter + proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.serializeBinaryToWriter ); } - f = message.getStopConditions(); + f = message.getStartingExecutions(); if (f != null) { writer.writeMessage( 2, f, - proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.serializeBinaryToWriter + proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.serializeBinaryToWriter + ); + } + f = /** @type {number} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeInt64( + 3, + f + ); + } + f = /** @type {!proto.ml_metadata.LineageSubgraphQueryOptions.Direction} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeEnum( + 4, + f ); } }; +/** + * @enum {number} + */ +proto.ml_metadata.LineageSubgraphQueryOptions.Direction = { + DIRECTION_UNSPECIFIED: 0, + UPSTREAM: 1, + DOWNSTREAM: 2, + BIDIRECTIONAL: 3 +}; + @@ -12005,8 +14221,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.toObject(opt_includeInstance, this); +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.toObject(opt_includeInstance, this); }; @@ -12015,15 +14231,13 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.toObject * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} msg The msg instance to transform. + * @param {!proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.toObject = function(includeInstance, msg) { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.toObject = function(includeInstance, msg) { var f, obj = { - maxNumHops: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - boundaryArtifacts: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - boundaryExecutions: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f + filterQuery: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f }; if (includeInstance) { @@ -12037,23 +14251,23 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.toObject = functio /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinary = function(bytes) { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint; - return proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes; + return proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} msg The message object to deserialize into. + * @param {!proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -12061,16 +14275,8 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryF var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setMaxNumHops(value); - break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setBoundaryArtifacts(value); - break; - case 3: var value = /** @type {string} */ (reader.readString()); - msg.setBoundaryExecutions(value); + msg.setFilterQuery(value); break; default: reader.skipField(); @@ -12085,9 +14291,9 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.deserializeBinaryF * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.serializeBinary = function() { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.serializeBinaryToWriter(this, writer); + proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -12095,30 +14301,16 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.serializ /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} message + * @param {!proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( - 1, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { writer.writeString( - 3, + 1, f ); } @@ -12126,28 +14318,28 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.serializeBinaryToW /** - * optional int64 max_num_hops = 1; - * @return {number} + * optional string filter_query = 1; + * @return {string} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.getMaxNumHops = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.prototype.getFilterQuery = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {number} value - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this + * @param {string} value + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} returns this */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.setMaxNumHops = function(value) { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.prototype.setFilterQuery = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} returns this */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearMaxNumHops = function() { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.prototype.clearFilterQuery = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -12156,35 +14348,36 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearMax * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.hasMaxNumHops = function() { +proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes.prototype.hasFilterQuery = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional string boundary_artifacts = 2; - * @return {string} + * optional StartingNodes starting_artifacts = 1; + * @return {?proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.getBoundaryArtifacts = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.getStartingArtifacts = function() { + return /** @type{?proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes, 1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this - */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.setBoundaryArtifacts = function(value) { - return jspb.Message.setField(this, 2, value); + * @param {?proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes|undefined} value + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this +*/ +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.setStartingArtifacts = function(value) { + return jspb.Message.setOneofWrapperField(this, 1, proto.ml_metadata.LineageSubgraphQueryOptions.oneofGroups_[0], value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearBoundaryArtifacts = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.clearStartingArtifacts = function() { + return this.setStartingArtifacts(undefined); }; @@ -12192,35 +14385,36 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearBou * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.hasBoundaryArtifacts = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.hasStartingArtifacts = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional string boundary_executions = 3; - * @return {string} + * optional StartingNodes starting_executions = 2; + * @return {?proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.getBoundaryExecutions = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.getStartingExecutions = function() { + return /** @type{?proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes, 2)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this - */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.setBoundaryExecutions = function(value) { - return jspb.Message.setField(this, 3, value); + * @param {?proto.ml_metadata.LineageSubgraphQueryOptions.StartingNodes|undefined} value + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this +*/ +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.setStartingExecutions = function(value) { + return jspb.Message.setOneofWrapperField(this, 2, proto.ml_metadata.LineageSubgraphQueryOptions.oneofGroups_[0], value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearBoundaryExecutions = function() { - return jspb.Message.setField(this, 3, undefined); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.clearStartingExecutions = function() { + return this.setStartingExecutions(undefined); }; @@ -12228,36 +14422,35 @@ proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.clearBou * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint.prototype.hasBoundaryExecutions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.hasStartingExecutions = function() { + return jspb.Message.getField(this, 2) != null; }; /** - * optional ListOperationOptions artifacts_options = 1; - * @return {?proto.ml_metadata.ListOperationOptions} + * optional int64 max_num_hops = 3; + * @return {number} */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.getArtifactsOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.ListOperationOptions, 1)); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.getMaxNumHops = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 3, 0)); }; /** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this -*/ -proto.ml_metadata.LineageGraphQueryOptions.prototype.setArtifactsOptions = function(value) { - return jspb.Message.setOneofWrapperField(this, 1, proto.ml_metadata.LineageGraphQueryOptions.oneofGroups_[0], value); + * @param {number} value + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this + */ +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.setMaxNumHops = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.clearArtifactsOptions = function() { - return this.setArtifactsOptions(undefined); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.clearMaxNumHops = function() { + return jspb.Message.setField(this, 3, undefined); }; @@ -12265,36 +14458,35 @@ proto.ml_metadata.LineageGraphQueryOptions.prototype.clearArtifactsOptions = fun * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.hasArtifactsOptions = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.hasMaxNumHops = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * optional BoundaryConstraint stop_conditions = 2; - * @return {?proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} + * optional Direction direction = 4; + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions.Direction} */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.getStopConditions = function() { - return /** @type{?proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint, 2)); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.getDirection = function() { + return /** @type {!proto.ml_metadata.LineageSubgraphQueryOptions.Direction} */ (jspb.Message.getFieldWithDefault(this, 4, 0)); }; /** - * @param {?proto.ml_metadata.LineageGraphQueryOptions.BoundaryConstraint|undefined} value - * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this -*/ -proto.ml_metadata.LineageGraphQueryOptions.prototype.setStopConditions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); + * @param {!proto.ml_metadata.LineageSubgraphQueryOptions.Direction} value + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this + */ +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.setDirection = function(value) { + return jspb.Message.setField(this, 4, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.LineageGraphQueryOptions} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.LineageSubgraphQueryOptions} returns this */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.clearStopConditions = function() { - return this.setStopConditions(undefined); +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.clearDirection = function() { + return jspb.Message.setField(this, 4, undefined); }; @@ -12302,8 +14494,8 @@ proto.ml_metadata.LineageGraphQueryOptions.prototype.clearStopConditions = funct * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.LineageGraphQueryOptions.prototype.hasStopConditions = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.LineageSubgraphQueryOptions.prototype.hasDirection = function() { + return jspb.Message.getField(this, 4) != null; }; @@ -12315,7 +14507,9 @@ proto.ml_metadata.PropertyType = { INT: 1, DOUBLE: 2, STRING: 3, - STRUCT: 4 + STRUCT: 4, + PROTO: 5, + BOOLEAN: 6 }; diff --git a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.d.ts b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.d.ts index 506a36e70e..1489129c23 100644 --- a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.d.ts +++ b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.d.ts @@ -64,6 +64,13 @@ export class MetadataStoreServiceClient { response: ml_metadata_proto_metadata_store_service_pb.PutExecutionResponse) => void ): grpcWeb.ClientReadableStream; + putLineageSubgraph( + request: ml_metadata_proto_metadata_store_service_pb.PutLineageSubgraphRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.PutLineageSubgraphResponse) => void + ): grpcWeb.ClientReadableStream; + putContexts( request: ml_metadata_proto_metadata_store_service_pb.PutContextsRequest, metadata: grpcWeb.Metadata | undefined, @@ -253,6 +260,48 @@ export class MetadataStoreServiceClient { response: ml_metadata_proto_metadata_store_service_pb.GetEventsByArtifactIDsResponse) => void ): grpcWeb.ClientReadableStream; + getArtifactsByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetArtifactsByExternalIdsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetArtifactsByExternalIdsResponse) => void + ): grpcWeb.ClientReadableStream; + + getExecutionsByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetExecutionsByExternalIdsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetExecutionsByExternalIdsResponse) => void + ): grpcWeb.ClientReadableStream; + + getContextsByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetContextsByExternalIdsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetContextsByExternalIdsResponse) => void + ): grpcWeb.ClientReadableStream; + + getArtifactTypesByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetArtifactTypesByExternalIdsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetArtifactTypesByExternalIdsResponse) => void + ): grpcWeb.ClientReadableStream; + + getExecutionTypesByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetExecutionTypesByExternalIdsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetExecutionTypesByExternalIdsResponse) => void + ): grpcWeb.ClientReadableStream; + + getContextTypesByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetContextTypesByExternalIdsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetContextTypesByExternalIdsResponse) => void + ): grpcWeb.ClientReadableStream; + getContextsByArtifact( request: ml_metadata_proto_metadata_store_service_pb.GetContextsByArtifactRequest, metadata: grpcWeb.Metadata | undefined, @@ -281,6 +330,20 @@ export class MetadataStoreServiceClient { response: ml_metadata_proto_metadata_store_service_pb.GetChildrenContextsByContextResponse) => void ): grpcWeb.ClientReadableStream; + getParentContextsByContexts( + request: ml_metadata_proto_metadata_store_service_pb.GetParentContextsByContextsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetParentContextsByContextsResponse) => void + ): grpcWeb.ClientReadableStream; + + getChildrenContextsByContexts( + request: ml_metadata_proto_metadata_store_service_pb.GetChildrenContextsByContextsRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetChildrenContextsByContextsResponse) => void + ): grpcWeb.ClientReadableStream; + getArtifactsByContext( request: ml_metadata_proto_metadata_store_service_pb.GetArtifactsByContextRequest, metadata: grpcWeb.Metadata | undefined, @@ -302,6 +365,13 @@ export class MetadataStoreServiceClient { response: ml_metadata_proto_metadata_store_service_pb.GetLineageGraphResponse) => void ): grpcWeb.ClientReadableStream; + getLineageSubgraph( + request: ml_metadata_proto_metadata_store_service_pb.GetLineageSubgraphRequest, + metadata: grpcWeb.Metadata | undefined, + callback: (err: grpcWeb.RpcError, + response: ml_metadata_proto_metadata_store_service_pb.GetLineageSubgraphResponse) => void + ): grpcWeb.ClientReadableStream; + } export class MetadataStoreServicePromiseClient { @@ -349,6 +419,11 @@ export class MetadataStoreServicePromiseClient { metadata?: grpcWeb.Metadata ): Promise; + putLineageSubgraph( + request: ml_metadata_proto_metadata_store_service_pb.PutLineageSubgraphRequest, + metadata?: grpcWeb.Metadata + ): Promise; + putContexts( request: ml_metadata_proto_metadata_store_service_pb.PutContextsRequest, metadata?: grpcWeb.Metadata @@ -484,6 +559,36 @@ export class MetadataStoreServicePromiseClient { metadata?: grpcWeb.Metadata ): Promise; + getArtifactsByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetArtifactsByExternalIdsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + + getExecutionsByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetExecutionsByExternalIdsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + + getContextsByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetContextsByExternalIdsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + + getArtifactTypesByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetArtifactTypesByExternalIdsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + + getExecutionTypesByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetExecutionTypesByExternalIdsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + + getContextTypesByExternalIds( + request: ml_metadata_proto_metadata_store_service_pb.GetContextTypesByExternalIdsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + getContextsByArtifact( request: ml_metadata_proto_metadata_store_service_pb.GetContextsByArtifactRequest, metadata?: grpcWeb.Metadata @@ -504,6 +609,16 @@ export class MetadataStoreServicePromiseClient { metadata?: grpcWeb.Metadata ): Promise; + getParentContextsByContexts( + request: ml_metadata_proto_metadata_store_service_pb.GetParentContextsByContextsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + + getChildrenContextsByContexts( + request: ml_metadata_proto_metadata_store_service_pb.GetChildrenContextsByContextsRequest, + metadata?: grpcWeb.Metadata + ): Promise; + getArtifactsByContext( request: ml_metadata_proto_metadata_store_service_pb.GetArtifactsByContextRequest, metadata?: grpcWeb.Metadata @@ -519,5 +634,10 @@ export class MetadataStoreServicePromiseClient { metadata?: grpcWeb.Metadata ): Promise; + getLineageSubgraph( + request: ml_metadata_proto_metadata_store_service_pb.GetLineageSubgraphRequest, + metadata?: grpcWeb.Metadata + ): Promise; + } diff --git a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.js b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.js index 9e4d1a33d2..db5c216b49 100644 --- a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.js +++ b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_grpc_web_pb.js @@ -16,6 +16,8 @@ const grpc = {}; grpc.web = require('grpc-web'); +var google_protobuf_field_mask_pb = require('google-protobuf/google/protobuf/field_mask_pb.js') + var ml_metadata_proto_metadata_store_pb = require('../../ml_metadata/proto/metadata_store_pb.js') const proto = {}; proto.ml_metadata = require('./metadata_store_service_pb.js'); @@ -560,6 +562,67 @@ proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.putExecution = }; +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.PutLineageSubgraphRequest, + * !proto.ml_metadata.PutLineageSubgraphResponse>} + */ +const methodDescriptor_MetadataStoreService_PutLineageSubgraph = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/PutLineageSubgraph', + grpc.web.MethodType.UNARY, + proto.ml_metadata.PutLineageSubgraphRequest, + proto.ml_metadata.PutLineageSubgraphResponse, + /** + * @param {!proto.ml_metadata.PutLineageSubgraphRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.PutLineageSubgraphResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.PutLineageSubgraphRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.PutLineageSubgraphResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.putLineageSubgraph = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/PutLineageSubgraph', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_PutLineageSubgraph, + callback); +}; + + +/** + * @param {!proto.ml_metadata.PutLineageSubgraphRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.putLineageSubgraph = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/PutLineageSubgraph', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_PutLineageSubgraph); +}; + + /** * @const * @type {!grpc.web.MethodDescriptor< @@ -2207,6 +2270,372 @@ proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getEventsByArtifac }; +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetArtifactsByExternalIdsRequest, + * !proto.ml_metadata.GetArtifactsByExternalIdsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetArtifactsByExternalIds = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetArtifactsByExternalIds', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetArtifactsByExternalIdsRequest, + proto.ml_metadata.GetArtifactsByExternalIdsResponse, + /** + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetArtifactsByExternalIdsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetArtifactsByExternalIdsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getArtifactsByExternalIds = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetArtifactsByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetArtifactsByExternalIds, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getArtifactsByExternalIds = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetArtifactsByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetArtifactsByExternalIds); +}; + + +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetExecutionsByExternalIdsRequest, + * !proto.ml_metadata.GetExecutionsByExternalIdsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetExecutionsByExternalIds = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetExecutionsByExternalIds', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetExecutionsByExternalIdsRequest, + proto.ml_metadata.GetExecutionsByExternalIdsResponse, + /** + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetExecutionsByExternalIdsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetExecutionsByExternalIdsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getExecutionsByExternalIds = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetExecutionsByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetExecutionsByExternalIds, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getExecutionsByExternalIds = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetExecutionsByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetExecutionsByExternalIds); +}; + + +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetContextsByExternalIdsRequest, + * !proto.ml_metadata.GetContextsByExternalIdsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetContextsByExternalIds = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetContextsByExternalIds', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetContextsByExternalIdsRequest, + proto.ml_metadata.GetContextsByExternalIdsResponse, + /** + * @param {!proto.ml_metadata.GetContextsByExternalIdsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetContextsByExternalIdsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetContextsByExternalIdsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetContextsByExternalIdsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getContextsByExternalIds = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetContextsByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetContextsByExternalIds, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetContextsByExternalIdsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getContextsByExternalIds = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetContextsByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetContextsByExternalIds); +}; + + +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetArtifactTypesByExternalIdsRequest, + * !proto.ml_metadata.GetArtifactTypesByExternalIdsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetArtifactTypesByExternalIds = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetArtifactTypesByExternalIds', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetArtifactTypesByExternalIdsRequest, + proto.ml_metadata.GetArtifactTypesByExternalIdsResponse, + /** + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetArtifactTypesByExternalIdsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getArtifactTypesByExternalIds = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetArtifactTypesByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetArtifactTypesByExternalIds, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getArtifactTypesByExternalIds = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetArtifactTypesByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetArtifactTypesByExternalIds); +}; + + +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetExecutionTypesByExternalIdsRequest, + * !proto.ml_metadata.GetExecutionTypesByExternalIdsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetExecutionTypesByExternalIds = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetExecutionTypesByExternalIds', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetExecutionTypesByExternalIdsRequest, + proto.ml_metadata.GetExecutionTypesByExternalIdsResponse, + /** + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetExecutionTypesByExternalIdsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getExecutionTypesByExternalIds = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetExecutionTypesByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetExecutionTypesByExternalIds, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getExecutionTypesByExternalIds = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetExecutionTypesByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetExecutionTypesByExternalIds); +}; + + +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetContextTypesByExternalIdsRequest, + * !proto.ml_metadata.GetContextTypesByExternalIdsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetContextTypesByExternalIds = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetContextTypesByExternalIds', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetContextTypesByExternalIdsRequest, + proto.ml_metadata.GetContextTypesByExternalIdsResponse, + /** + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetContextTypesByExternalIdsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetContextTypesByExternalIdsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getContextTypesByExternalIds = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetContextTypesByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetContextTypesByExternalIds, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getContextTypesByExternalIds = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetContextTypesByExternalIds', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetContextTypesByExternalIds); +}; + + /** * @const * @type {!grpc.web.MethodDescriptor< @@ -2451,6 +2880,128 @@ proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getChildrenContext }; +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetParentContextsByContextsRequest, + * !proto.ml_metadata.GetParentContextsByContextsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetParentContextsByContexts = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetParentContextsByContexts', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetParentContextsByContextsRequest, + proto.ml_metadata.GetParentContextsByContextsResponse, + /** + * @param {!proto.ml_metadata.GetParentContextsByContextsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetParentContextsByContextsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetParentContextsByContextsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetParentContextsByContextsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getParentContextsByContexts = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetParentContextsByContexts', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetParentContextsByContexts, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetParentContextsByContextsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getParentContextsByContexts = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetParentContextsByContexts', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetParentContextsByContexts); +}; + + +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetChildrenContextsByContextsRequest, + * !proto.ml_metadata.GetChildrenContextsByContextsResponse>} + */ +const methodDescriptor_MetadataStoreService_GetChildrenContextsByContexts = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetChildrenContextsByContexts', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetChildrenContextsByContextsRequest, + proto.ml_metadata.GetChildrenContextsByContextsResponse, + /** + * @param {!proto.ml_metadata.GetChildrenContextsByContextsRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetChildrenContextsByContextsResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetChildrenContextsByContextsRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetChildrenContextsByContextsResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getChildrenContextsByContexts = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetChildrenContextsByContexts', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetChildrenContextsByContexts, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetChildrenContextsByContextsRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getChildrenContextsByContexts = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetChildrenContextsByContexts', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetChildrenContextsByContexts); +}; + + /** * @const * @type {!grpc.web.MethodDescriptor< @@ -2634,5 +3185,66 @@ proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getLineageGraph = }; +/** + * @const + * @type {!grpc.web.MethodDescriptor< + * !proto.ml_metadata.GetLineageSubgraphRequest, + * !proto.ml_metadata.GetLineageSubgraphResponse>} + */ +const methodDescriptor_MetadataStoreService_GetLineageSubgraph = new grpc.web.MethodDescriptor( + '/ml_metadata.MetadataStoreService/GetLineageSubgraph', + grpc.web.MethodType.UNARY, + proto.ml_metadata.GetLineageSubgraphRequest, + proto.ml_metadata.GetLineageSubgraphResponse, + /** + * @param {!proto.ml_metadata.GetLineageSubgraphRequest} request + * @return {!Uint8Array} + */ + function(request) { + return request.serializeBinary(); + }, + proto.ml_metadata.GetLineageSubgraphResponse.deserializeBinary +); + + +/** + * @param {!proto.ml_metadata.GetLineageSubgraphRequest} request The + * request proto + * @param {?Object} metadata User defined + * call metadata + * @param {function(?grpc.web.RpcError, ?proto.ml_metadata.GetLineageSubgraphResponse)} + * callback The callback function(error, response) + * @return {!grpc.web.ClientReadableStream|undefined} + * The XHR Node Readable Stream + */ +proto.ml_metadata.MetadataStoreServiceClient.prototype.getLineageSubgraph = + function(request, metadata, callback) { + return this.client_.rpcCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetLineageSubgraph', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetLineageSubgraph, + callback); +}; + + +/** + * @param {!proto.ml_metadata.GetLineageSubgraphRequest} request The + * request proto + * @param {?Object=} metadata User defined + * call metadata + * @return {!Promise} + * Promise that resolves to the response + */ +proto.ml_metadata.MetadataStoreServicePromiseClient.prototype.getLineageSubgraph = + function(request, metadata) { + return this.client_.unaryCall(this.hostname_ + + '/ml_metadata.MetadataStoreService/GetLineageSubgraph', + request, + metadata || {}, + methodDescriptor_MetadataStoreService_GetLineageSubgraph); +}; + + module.exports = proto.ml_metadata; diff --git a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.d.ts b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.d.ts index d59fdebc00..83fb1f015c 100644 --- a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.d.ts +++ b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.d.ts @@ -1,5 +1,6 @@ import * as jspb from 'google-protobuf' +import * as google_protobuf_field_mask_pb from 'google-protobuf/google/protobuf/field_mask_pb'; import * as ml_metadata_proto_metadata_store_pb from '../../ml_metadata/proto/metadata_store_pb'; @@ -119,6 +120,16 @@ export class PutArtifactsRequest extends jspb.Message { hasOptions(): boolean; clearOptions(): PutArtifactsRequest; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutArtifactsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutArtifactsRequest; + + getUpdateMask(): google_protobuf_field_mask_pb.FieldMask | undefined; + setUpdateMask(value?: google_protobuf_field_mask_pb.FieldMask): PutArtifactsRequest; + hasUpdateMask(): boolean; + clearUpdateMask(): PutArtifactsRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutArtifactsRequest.AsObject; static toObject(includeInstance: boolean, msg: PutArtifactsRequest): PutArtifactsRequest.AsObject; @@ -131,6 +142,8 @@ export namespace PutArtifactsRequest { export type AsObject = { artifactsList: Array, options?: PutArtifactsRequest.Options.AsObject, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + updateMask?: google_protobuf_field_mask_pb.FieldMask.AsObject, } export class Options extends jspb.Message { @@ -191,6 +204,11 @@ export class PutArtifactTypeRequest extends jspb.Message { getAllFieldsMatch(): boolean; setAllFieldsMatch(value: boolean): PutArtifactTypeRequest; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutArtifactTypeRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutArtifactTypeRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutArtifactTypeRequest.AsObject; static toObject(includeInstance: boolean, msg: PutArtifactTypeRequest): PutArtifactTypeRequest.AsObject; @@ -206,6 +224,7 @@ export namespace PutArtifactTypeRequest { canOmitFields: boolean, canDeleteFields: boolean, allFieldsMatch: boolean, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -233,6 +252,16 @@ export class PutExecutionsRequest extends jspb.Message { clearExecutionsList(): PutExecutionsRequest; addExecutions(value?: ml_metadata_proto_metadata_store_pb.Execution, index?: number): ml_metadata_proto_metadata_store_pb.Execution; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutExecutionsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutExecutionsRequest; + + getUpdateMask(): google_protobuf_field_mask_pb.FieldMask | undefined; + setUpdateMask(value?: google_protobuf_field_mask_pb.FieldMask): PutExecutionsRequest; + hasUpdateMask(): boolean; + clearUpdateMask(): PutExecutionsRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutExecutionsRequest.AsObject; static toObject(includeInstance: boolean, msg: PutExecutionsRequest): PutExecutionsRequest.AsObject; @@ -244,6 +273,8 @@ export class PutExecutionsRequest extends jspb.Message { export namespace PutExecutionsRequest { export type AsObject = { executionsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + updateMask?: google_protobuf_field_mask_pb.FieldMask.AsObject, } } @@ -285,6 +316,11 @@ export class PutExecutionTypeRequest extends jspb.Message { getAllFieldsMatch(): boolean; setAllFieldsMatch(value: boolean): PutExecutionTypeRequest; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutExecutionTypeRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutExecutionTypeRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutExecutionTypeRequest.AsObject; static toObject(includeInstance: boolean, msg: PutExecutionTypeRequest): PutExecutionTypeRequest.AsObject; @@ -300,6 +336,7 @@ export namespace PutExecutionTypeRequest { canOmitFields: boolean, canDeleteFields: boolean, allFieldsMatch: boolean, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -327,6 +364,11 @@ export class PutEventsRequest extends jspb.Message { clearEventsList(): PutEventsRequest; addEvents(value?: ml_metadata_proto_metadata_store_pb.Event, index?: number): ml_metadata_proto_metadata_store_pb.Event; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutEventsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutEventsRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutEventsRequest.AsObject; static toObject(includeInstance: boolean, msg: PutEventsRequest): PutEventsRequest.AsObject; @@ -338,6 +380,7 @@ export class PutEventsRequest extends jspb.Message { export namespace PutEventsRequest { export type AsObject = { eventsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -376,6 +419,11 @@ export class PutExecutionRequest extends jspb.Message { hasOptions(): boolean; clearOptions(): PutExecutionRequest; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutExecutionRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutExecutionRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutExecutionRequest.AsObject; static toObject(includeInstance: boolean, msg: PutExecutionRequest): PutExecutionRequest.AsObject; @@ -390,6 +438,7 @@ export namespace PutExecutionRequest { artifactEventPairsList: Array, contextsList: Array, options?: PutExecutionRequest.Options.AsObject, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } export class ArtifactAndEvent extends jspb.Message { @@ -423,6 +472,9 @@ export namespace PutExecutionRequest { getReuseContextIfAlreadyExist(): boolean; setReuseContextIfAlreadyExist(value: boolean): Options; + getReuseArtifactIfAlreadyExistByExternalId(): boolean; + setReuseArtifactIfAlreadyExistByExternalId(value: boolean): Options; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): Options.AsObject; static toObject(includeInstance: boolean, msg: Options): Options.AsObject; @@ -434,6 +486,7 @@ export namespace PutExecutionRequest { export namespace Options { export type AsObject = { reuseContextIfAlreadyExist: boolean, + reuseArtifactIfAlreadyExistByExternalId: boolean, } } @@ -469,6 +522,140 @@ export namespace PutExecutionResponse { } } +export class PutLineageSubgraphRequest extends jspb.Message { + getExecutionsList(): Array; + setExecutionsList(value: Array): PutLineageSubgraphRequest; + clearExecutionsList(): PutLineageSubgraphRequest; + addExecutions(value?: ml_metadata_proto_metadata_store_pb.Execution, index?: number): ml_metadata_proto_metadata_store_pb.Execution; + + getArtifactsList(): Array; + setArtifactsList(value: Array): PutLineageSubgraphRequest; + clearArtifactsList(): PutLineageSubgraphRequest; + addArtifacts(value?: ml_metadata_proto_metadata_store_pb.Artifact, index?: number): ml_metadata_proto_metadata_store_pb.Artifact; + + getContextsList(): Array; + setContextsList(value: Array): PutLineageSubgraphRequest; + clearContextsList(): PutLineageSubgraphRequest; + addContexts(value?: ml_metadata_proto_metadata_store_pb.Context, index?: number): ml_metadata_proto_metadata_store_pb.Context; + + getEventEdgesList(): Array; + setEventEdgesList(value: Array): PutLineageSubgraphRequest; + clearEventEdgesList(): PutLineageSubgraphRequest; + addEventEdges(value?: PutLineageSubgraphRequest.EventEdge, index?: number): PutLineageSubgraphRequest.EventEdge; + + getOptions(): PutLineageSubgraphRequest.Options | undefined; + setOptions(value?: PutLineageSubgraphRequest.Options): PutLineageSubgraphRequest; + hasOptions(): boolean; + clearOptions(): PutLineageSubgraphRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutLineageSubgraphRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutLineageSubgraphRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): PutLineageSubgraphRequest.AsObject; + static toObject(includeInstance: boolean, msg: PutLineageSubgraphRequest): PutLineageSubgraphRequest.AsObject; + static serializeBinaryToWriter(message: PutLineageSubgraphRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): PutLineageSubgraphRequest; + static deserializeBinaryFromReader(message: PutLineageSubgraphRequest, reader: jspb.BinaryReader): PutLineageSubgraphRequest; +} + +export namespace PutLineageSubgraphRequest { + export type AsObject = { + executionsList: Array, + artifactsList: Array, + contextsList: Array, + eventEdgesList: Array, + options?: PutLineageSubgraphRequest.Options.AsObject, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } + + export class EventEdge extends jspb.Message { + getExecutionIndex(): number; + setExecutionIndex(value: number): EventEdge; + + getArtifactIndex(): number; + setArtifactIndex(value: number): EventEdge; + + getEvent(): ml_metadata_proto_metadata_store_pb.Event | undefined; + setEvent(value?: ml_metadata_proto_metadata_store_pb.Event): EventEdge; + hasEvent(): boolean; + clearEvent(): EventEdge; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): EventEdge.AsObject; + static toObject(includeInstance: boolean, msg: EventEdge): EventEdge.AsObject; + static serializeBinaryToWriter(message: EventEdge, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): EventEdge; + static deserializeBinaryFromReader(message: EventEdge, reader: jspb.BinaryReader): EventEdge; + } + + export namespace EventEdge { + export type AsObject = { + executionIndex: number, + artifactIndex: number, + event?: ml_metadata_proto_metadata_store_pb.Event.AsObject, + } + } + + + export class Options extends jspb.Message { + getReuseContextIfAlreadyExist(): boolean; + setReuseContextIfAlreadyExist(value: boolean): Options; + + getReuseArtifactIfAlreadyExistByExternalId(): boolean; + setReuseArtifactIfAlreadyExistByExternalId(value: boolean): Options; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): Options.AsObject; + static toObject(includeInstance: boolean, msg: Options): Options.AsObject; + static serializeBinaryToWriter(message: Options, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): Options; + static deserializeBinaryFromReader(message: Options, reader: jspb.BinaryReader): Options; + } + + export namespace Options { + export type AsObject = { + reuseContextIfAlreadyExist: boolean, + reuseArtifactIfAlreadyExistByExternalId: boolean, + } + } + +} + +export class PutLineageSubgraphResponse extends jspb.Message { + getExecutionIdsList(): Array; + setExecutionIdsList(value: Array): PutLineageSubgraphResponse; + clearExecutionIdsList(): PutLineageSubgraphResponse; + addExecutionIds(value: number, index?: number): PutLineageSubgraphResponse; + + getArtifactIdsList(): Array; + setArtifactIdsList(value: Array): PutLineageSubgraphResponse; + clearArtifactIdsList(): PutLineageSubgraphResponse; + addArtifactIds(value: number, index?: number): PutLineageSubgraphResponse; + + getContextIdsList(): Array; + setContextIdsList(value: Array): PutLineageSubgraphResponse; + clearContextIdsList(): PutLineageSubgraphResponse; + addContextIds(value: number, index?: number): PutLineageSubgraphResponse; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): PutLineageSubgraphResponse.AsObject; + static toObject(includeInstance: boolean, msg: PutLineageSubgraphResponse): PutLineageSubgraphResponse.AsObject; + static serializeBinaryToWriter(message: PutLineageSubgraphResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): PutLineageSubgraphResponse; + static deserializeBinaryFromReader(message: PutLineageSubgraphResponse, reader: jspb.BinaryReader): PutLineageSubgraphResponse; +} + +export namespace PutLineageSubgraphResponse { + export type AsObject = { + executionIdsList: Array, + artifactIdsList: Array, + contextIdsList: Array, + } +} + export class PutTypesRequest extends jspb.Message { getArtifactTypesList(): Array; setArtifactTypesList(value: Array): PutTypesRequest; @@ -497,6 +684,11 @@ export class PutTypesRequest extends jspb.Message { getAllFieldsMatch(): boolean; setAllFieldsMatch(value: boolean): PutTypesRequest; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutTypesRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutTypesRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutTypesRequest.AsObject; static toObject(includeInstance: boolean, msg: PutTypesRequest): PutTypesRequest.AsObject; @@ -514,6 +706,7 @@ export namespace PutTypesRequest { canOmitFields: boolean, canDeleteFields: boolean, allFieldsMatch: boolean, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -567,6 +760,11 @@ export class PutContextTypeRequest extends jspb.Message { getAllFieldsMatch(): boolean; setAllFieldsMatch(value: boolean): PutContextTypeRequest; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutContextTypeRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutContextTypeRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutContextTypeRequest.AsObject; static toObject(includeInstance: boolean, msg: PutContextTypeRequest): PutContextTypeRequest.AsObject; @@ -582,6 +780,7 @@ export namespace PutContextTypeRequest { canOmitFields: boolean, canDeleteFields: boolean, allFieldsMatch: boolean, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -609,6 +808,16 @@ export class PutContextsRequest extends jspb.Message { clearContextsList(): PutContextsRequest; addContexts(value?: ml_metadata_proto_metadata_store_pb.Context, index?: number): ml_metadata_proto_metadata_store_pb.Context; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutContextsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutContextsRequest; + + getUpdateMask(): google_protobuf_field_mask_pb.FieldMask | undefined; + setUpdateMask(value?: google_protobuf_field_mask_pb.FieldMask): PutContextsRequest; + hasUpdateMask(): boolean; + clearUpdateMask(): PutContextsRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutContextsRequest.AsObject; static toObject(includeInstance: boolean, msg: PutContextsRequest): PutContextsRequest.AsObject; @@ -620,6 +829,8 @@ export class PutContextsRequest extends jspb.Message { export namespace PutContextsRequest { export type AsObject = { contextsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + updateMask?: google_protobuf_field_mask_pb.FieldMask.AsObject, } } @@ -654,6 +865,11 @@ export class PutAttributionsAndAssociationsRequest extends jspb.Message { clearAssociationsList(): PutAttributionsAndAssociationsRequest; addAssociations(value?: ml_metadata_proto_metadata_store_pb.Association, index?: number): ml_metadata_proto_metadata_store_pb.Association; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutAttributionsAndAssociationsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutAttributionsAndAssociationsRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutAttributionsAndAssociationsRequest.AsObject; static toObject(includeInstance: boolean, msg: PutAttributionsAndAssociationsRequest): PutAttributionsAndAssociationsRequest.AsObject; @@ -666,6 +882,7 @@ export namespace PutAttributionsAndAssociationsRequest { export type AsObject = { attributionsList: Array, associationsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -689,6 +906,11 @@ export class PutParentContextsRequest extends jspb.Message { clearParentContextsList(): PutParentContextsRequest; addParentContexts(value?: ml_metadata_proto_metadata_store_pb.ParentContext, index?: number): ml_metadata_proto_metadata_store_pb.ParentContext; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): PutParentContextsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): PutParentContextsRequest; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PutParentContextsRequest.AsObject; static toObject(includeInstance: boolean, msg: PutParentContextsRequest): PutParentContextsRequest.AsObject; @@ -700,6 +922,7 @@ export class PutParentContextsRequest extends jspb.Message { export namespace PutParentContextsRequest { export type AsObject = { parentContextsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -833,6 +1056,9 @@ export class GetArtifactsByIDRequest extends jspb.Message { clearArtifactIdsList(): GetArtifactsByIDRequest; addArtifactIds(value: number, index?: number): GetArtifactsByIDRequest; + getPopulateArtifactTypes(): boolean; + setPopulateArtifactTypes(value: boolean): GetArtifactsByIDRequest; + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetArtifactsByIDRequest; hasTransactionOptions(): boolean; @@ -849,6 +1075,7 @@ export class GetArtifactsByIDRequest extends jspb.Message { export namespace GetArtifactsByIDRequest { export type AsObject = { artifactIdsList: Array, + populateArtifactTypes: boolean, transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, } } @@ -859,6 +1086,11 @@ export class GetArtifactsByIDResponse extends jspb.Message { clearArtifactsList(): GetArtifactsByIDResponse; addArtifacts(value?: ml_metadata_proto_metadata_store_pb.Artifact, index?: number): ml_metadata_proto_metadata_store_pb.Artifact; + getArtifactTypesList(): Array; + setArtifactTypesList(value: Array): GetArtifactsByIDResponse; + clearArtifactTypesList(): GetArtifactsByIDResponse; + addArtifactTypes(value?: ml_metadata_proto_metadata_store_pb.ArtifactType, index?: number): ml_metadata_proto_metadata_store_pb.ArtifactType; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): GetArtifactsByIDResponse.AsObject; static toObject(includeInstance: boolean, msg: GetArtifactsByIDResponse): GetArtifactsByIDResponse.AsObject; @@ -870,6 +1102,7 @@ export class GetArtifactsByIDResponse extends jspb.Message { export namespace GetArtifactsByIDResponse { export type AsObject = { artifactsList: Array, + artifactTypesList: Array, } } @@ -1187,6 +1420,282 @@ export namespace GetContextTypesResponse { } } +export class GetArtifactsByExternalIdsRequest extends jspb.Message { + getExternalIdsList(): Array; + setExternalIdsList(value: Array): GetArtifactsByExternalIdsRequest; + clearExternalIdsList(): GetArtifactsByExternalIdsRequest; + addExternalIds(value: string, index?: number): GetArtifactsByExternalIdsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetArtifactsByExternalIdsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetArtifactsByExternalIdsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetArtifactsByExternalIdsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetArtifactsByExternalIdsRequest): GetArtifactsByExternalIdsRequest.AsObject; + static serializeBinaryToWriter(message: GetArtifactsByExternalIdsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetArtifactsByExternalIdsRequest; + static deserializeBinaryFromReader(message: GetArtifactsByExternalIdsRequest, reader: jspb.BinaryReader): GetArtifactsByExternalIdsRequest; +} + +export namespace GetArtifactsByExternalIdsRequest { + export type AsObject = { + externalIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetArtifactsByExternalIdsResponse extends jspb.Message { + getArtifactsList(): Array; + setArtifactsList(value: Array): GetArtifactsByExternalIdsResponse; + clearArtifactsList(): GetArtifactsByExternalIdsResponse; + addArtifacts(value?: ml_metadata_proto_metadata_store_pb.Artifact, index?: number): ml_metadata_proto_metadata_store_pb.Artifact; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetArtifactsByExternalIdsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetArtifactsByExternalIdsResponse): GetArtifactsByExternalIdsResponse.AsObject; + static serializeBinaryToWriter(message: GetArtifactsByExternalIdsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetArtifactsByExternalIdsResponse; + static deserializeBinaryFromReader(message: GetArtifactsByExternalIdsResponse, reader: jspb.BinaryReader): GetArtifactsByExternalIdsResponse; +} + +export namespace GetArtifactsByExternalIdsResponse { + export type AsObject = { + artifactsList: Array, + } +} + +export class GetExecutionsByExternalIdsRequest extends jspb.Message { + getExternalIdsList(): Array; + setExternalIdsList(value: Array): GetExecutionsByExternalIdsRequest; + clearExternalIdsList(): GetExecutionsByExternalIdsRequest; + addExternalIds(value: string, index?: number): GetExecutionsByExternalIdsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetExecutionsByExternalIdsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetExecutionsByExternalIdsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetExecutionsByExternalIdsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetExecutionsByExternalIdsRequest): GetExecutionsByExternalIdsRequest.AsObject; + static serializeBinaryToWriter(message: GetExecutionsByExternalIdsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetExecutionsByExternalIdsRequest; + static deserializeBinaryFromReader(message: GetExecutionsByExternalIdsRequest, reader: jspb.BinaryReader): GetExecutionsByExternalIdsRequest; +} + +export namespace GetExecutionsByExternalIdsRequest { + export type AsObject = { + externalIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetExecutionsByExternalIdsResponse extends jspb.Message { + getExecutionsList(): Array; + setExecutionsList(value: Array): GetExecutionsByExternalIdsResponse; + clearExecutionsList(): GetExecutionsByExternalIdsResponse; + addExecutions(value?: ml_metadata_proto_metadata_store_pb.Execution, index?: number): ml_metadata_proto_metadata_store_pb.Execution; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetExecutionsByExternalIdsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetExecutionsByExternalIdsResponse): GetExecutionsByExternalIdsResponse.AsObject; + static serializeBinaryToWriter(message: GetExecutionsByExternalIdsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetExecutionsByExternalIdsResponse; + static deserializeBinaryFromReader(message: GetExecutionsByExternalIdsResponse, reader: jspb.BinaryReader): GetExecutionsByExternalIdsResponse; +} + +export namespace GetExecutionsByExternalIdsResponse { + export type AsObject = { + executionsList: Array, + } +} + +export class GetContextsByExternalIdsRequest extends jspb.Message { + getExternalIdsList(): Array; + setExternalIdsList(value: Array): GetContextsByExternalIdsRequest; + clearExternalIdsList(): GetContextsByExternalIdsRequest; + addExternalIds(value: string, index?: number): GetContextsByExternalIdsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetContextsByExternalIdsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetContextsByExternalIdsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetContextsByExternalIdsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetContextsByExternalIdsRequest): GetContextsByExternalIdsRequest.AsObject; + static serializeBinaryToWriter(message: GetContextsByExternalIdsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetContextsByExternalIdsRequest; + static deserializeBinaryFromReader(message: GetContextsByExternalIdsRequest, reader: jspb.BinaryReader): GetContextsByExternalIdsRequest; +} + +export namespace GetContextsByExternalIdsRequest { + export type AsObject = { + externalIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetContextsByExternalIdsResponse extends jspb.Message { + getContextsList(): Array; + setContextsList(value: Array): GetContextsByExternalIdsResponse; + clearContextsList(): GetContextsByExternalIdsResponse; + addContexts(value?: ml_metadata_proto_metadata_store_pb.Context, index?: number): ml_metadata_proto_metadata_store_pb.Context; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetContextsByExternalIdsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetContextsByExternalIdsResponse): GetContextsByExternalIdsResponse.AsObject; + static serializeBinaryToWriter(message: GetContextsByExternalIdsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetContextsByExternalIdsResponse; + static deserializeBinaryFromReader(message: GetContextsByExternalIdsResponse, reader: jspb.BinaryReader): GetContextsByExternalIdsResponse; +} + +export namespace GetContextsByExternalIdsResponse { + export type AsObject = { + contextsList: Array, + } +} + +export class GetArtifactTypesByExternalIdsRequest extends jspb.Message { + getExternalIdsList(): Array; + setExternalIdsList(value: Array): GetArtifactTypesByExternalIdsRequest; + clearExternalIdsList(): GetArtifactTypesByExternalIdsRequest; + addExternalIds(value: string, index?: number): GetArtifactTypesByExternalIdsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetArtifactTypesByExternalIdsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetArtifactTypesByExternalIdsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetArtifactTypesByExternalIdsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetArtifactTypesByExternalIdsRequest): GetArtifactTypesByExternalIdsRequest.AsObject; + static serializeBinaryToWriter(message: GetArtifactTypesByExternalIdsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetArtifactTypesByExternalIdsRequest; + static deserializeBinaryFromReader(message: GetArtifactTypesByExternalIdsRequest, reader: jspb.BinaryReader): GetArtifactTypesByExternalIdsRequest; +} + +export namespace GetArtifactTypesByExternalIdsRequest { + export type AsObject = { + externalIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetArtifactTypesByExternalIdsResponse extends jspb.Message { + getArtifactTypesList(): Array; + setArtifactTypesList(value: Array): GetArtifactTypesByExternalIdsResponse; + clearArtifactTypesList(): GetArtifactTypesByExternalIdsResponse; + addArtifactTypes(value?: ml_metadata_proto_metadata_store_pb.ArtifactType, index?: number): ml_metadata_proto_metadata_store_pb.ArtifactType; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetArtifactTypesByExternalIdsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetArtifactTypesByExternalIdsResponse): GetArtifactTypesByExternalIdsResponse.AsObject; + static serializeBinaryToWriter(message: GetArtifactTypesByExternalIdsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetArtifactTypesByExternalIdsResponse; + static deserializeBinaryFromReader(message: GetArtifactTypesByExternalIdsResponse, reader: jspb.BinaryReader): GetArtifactTypesByExternalIdsResponse; +} + +export namespace GetArtifactTypesByExternalIdsResponse { + export type AsObject = { + artifactTypesList: Array, + } +} + +export class GetExecutionTypesByExternalIdsRequest extends jspb.Message { + getExternalIdsList(): Array; + setExternalIdsList(value: Array): GetExecutionTypesByExternalIdsRequest; + clearExternalIdsList(): GetExecutionTypesByExternalIdsRequest; + addExternalIds(value: string, index?: number): GetExecutionTypesByExternalIdsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetExecutionTypesByExternalIdsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetExecutionTypesByExternalIdsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetExecutionTypesByExternalIdsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetExecutionTypesByExternalIdsRequest): GetExecutionTypesByExternalIdsRequest.AsObject; + static serializeBinaryToWriter(message: GetExecutionTypesByExternalIdsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetExecutionTypesByExternalIdsRequest; + static deserializeBinaryFromReader(message: GetExecutionTypesByExternalIdsRequest, reader: jspb.BinaryReader): GetExecutionTypesByExternalIdsRequest; +} + +export namespace GetExecutionTypesByExternalIdsRequest { + export type AsObject = { + externalIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetExecutionTypesByExternalIdsResponse extends jspb.Message { + getExecutionTypesList(): Array; + setExecutionTypesList(value: Array): GetExecutionTypesByExternalIdsResponse; + clearExecutionTypesList(): GetExecutionTypesByExternalIdsResponse; + addExecutionTypes(value?: ml_metadata_proto_metadata_store_pb.ExecutionType, index?: number): ml_metadata_proto_metadata_store_pb.ExecutionType; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetExecutionTypesByExternalIdsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetExecutionTypesByExternalIdsResponse): GetExecutionTypesByExternalIdsResponse.AsObject; + static serializeBinaryToWriter(message: GetExecutionTypesByExternalIdsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetExecutionTypesByExternalIdsResponse; + static deserializeBinaryFromReader(message: GetExecutionTypesByExternalIdsResponse, reader: jspb.BinaryReader): GetExecutionTypesByExternalIdsResponse; +} + +export namespace GetExecutionTypesByExternalIdsResponse { + export type AsObject = { + executionTypesList: Array, + } +} + +export class GetContextTypesByExternalIdsRequest extends jspb.Message { + getExternalIdsList(): Array; + setExternalIdsList(value: Array): GetContextTypesByExternalIdsRequest; + clearExternalIdsList(): GetContextTypesByExternalIdsRequest; + addExternalIds(value: string, index?: number): GetContextTypesByExternalIdsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetContextTypesByExternalIdsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetContextTypesByExternalIdsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetContextTypesByExternalIdsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetContextTypesByExternalIdsRequest): GetContextTypesByExternalIdsRequest.AsObject; + static serializeBinaryToWriter(message: GetContextTypesByExternalIdsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetContextTypesByExternalIdsRequest; + static deserializeBinaryFromReader(message: GetContextTypesByExternalIdsRequest, reader: jspb.BinaryReader): GetContextTypesByExternalIdsRequest; +} + +export namespace GetContextTypesByExternalIdsRequest { + export type AsObject = { + externalIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetContextTypesByExternalIdsResponse extends jspb.Message { + getContextTypesList(): Array; + setContextTypesList(value: Array): GetContextTypesByExternalIdsResponse; + clearContextTypesList(): GetContextTypesByExternalIdsResponse; + addContextTypes(value?: ml_metadata_proto_metadata_store_pb.ContextType, index?: number): ml_metadata_proto_metadata_store_pb.ContextType; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetContextTypesByExternalIdsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetContextTypesByExternalIdsResponse): GetContextTypesByExternalIdsResponse.AsObject; + static serializeBinaryToWriter(message: GetContextTypesByExternalIdsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetContextTypesByExternalIdsResponse; + static deserializeBinaryFromReader(message: GetContextTypesByExternalIdsResponse, reader: jspb.BinaryReader): GetContextTypesByExternalIdsResponse; +} + +export namespace GetContextTypesByExternalIdsResponse { + export type AsObject = { + contextTypesList: Array, + } +} + export class GetExecutionsByTypeRequest extends jspb.Message { getTypeName(): string; setTypeName(value: string): GetExecutionsByTypeRequest; @@ -2051,6 +2560,136 @@ export namespace GetChildrenContextsByContextResponse { } } +export class GetParentContextsByContextsRequest extends jspb.Message { + getContextIdsList(): Array; + setContextIdsList(value: Array): GetParentContextsByContextsRequest; + clearContextIdsList(): GetParentContextsByContextsRequest; + addContextIds(value: number, index?: number): GetParentContextsByContextsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetParentContextsByContextsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetParentContextsByContextsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetParentContextsByContextsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetParentContextsByContextsRequest): GetParentContextsByContextsRequest.AsObject; + static serializeBinaryToWriter(message: GetParentContextsByContextsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetParentContextsByContextsRequest; + static deserializeBinaryFromReader(message: GetParentContextsByContextsRequest, reader: jspb.BinaryReader): GetParentContextsByContextsRequest; +} + +export namespace GetParentContextsByContextsRequest { + export type AsObject = { + contextIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetParentContextsByContextsResponse extends jspb.Message { + getContextsMap(): jspb.Map; + clearContextsMap(): GetParentContextsByContextsResponse; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetParentContextsByContextsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetParentContextsByContextsResponse): GetParentContextsByContextsResponse.AsObject; + static serializeBinaryToWriter(message: GetParentContextsByContextsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetParentContextsByContextsResponse; + static deserializeBinaryFromReader(message: GetParentContextsByContextsResponse, reader: jspb.BinaryReader): GetParentContextsByContextsResponse; +} + +export namespace GetParentContextsByContextsResponse { + export type AsObject = { + contextsMap: Array<[number, GetParentContextsByContextsResponse.ParentContextsPerChild.AsObject]>, + } + + export class ParentContextsPerChild extends jspb.Message { + getParentContextsList(): Array; + setParentContextsList(value: Array): ParentContextsPerChild; + clearParentContextsList(): ParentContextsPerChild; + addParentContexts(value?: ml_metadata_proto_metadata_store_pb.Context, index?: number): ml_metadata_proto_metadata_store_pb.Context; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): ParentContextsPerChild.AsObject; + static toObject(includeInstance: boolean, msg: ParentContextsPerChild): ParentContextsPerChild.AsObject; + static serializeBinaryToWriter(message: ParentContextsPerChild, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): ParentContextsPerChild; + static deserializeBinaryFromReader(message: ParentContextsPerChild, reader: jspb.BinaryReader): ParentContextsPerChild; + } + + export namespace ParentContextsPerChild { + export type AsObject = { + parentContextsList: Array, + } + } + +} + +export class GetChildrenContextsByContextsRequest extends jspb.Message { + getContextIdsList(): Array; + setContextIdsList(value: Array): GetChildrenContextsByContextsRequest; + clearContextIdsList(): GetChildrenContextsByContextsRequest; + addContextIds(value: number, index?: number): GetChildrenContextsByContextsRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetChildrenContextsByContextsRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetChildrenContextsByContextsRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetChildrenContextsByContextsRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetChildrenContextsByContextsRequest): GetChildrenContextsByContextsRequest.AsObject; + static serializeBinaryToWriter(message: GetChildrenContextsByContextsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetChildrenContextsByContextsRequest; + static deserializeBinaryFromReader(message: GetChildrenContextsByContextsRequest, reader: jspb.BinaryReader): GetChildrenContextsByContextsRequest; +} + +export namespace GetChildrenContextsByContextsRequest { + export type AsObject = { + contextIdsList: Array, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetChildrenContextsByContextsResponse extends jspb.Message { + getContextsMap(): jspb.Map; + clearContextsMap(): GetChildrenContextsByContextsResponse; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetChildrenContextsByContextsResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetChildrenContextsByContextsResponse): GetChildrenContextsByContextsResponse.AsObject; + static serializeBinaryToWriter(message: GetChildrenContextsByContextsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetChildrenContextsByContextsResponse; + static deserializeBinaryFromReader(message: GetChildrenContextsByContextsResponse, reader: jspb.BinaryReader): GetChildrenContextsByContextsResponse; +} + +export namespace GetChildrenContextsByContextsResponse { + export type AsObject = { + contextsMap: Array<[number, GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.AsObject]>, + } + + export class ChildrenContextsPerParent extends jspb.Message { + getChildrenContextsList(): Array; + setChildrenContextsList(value: Array): ChildrenContextsPerParent; + clearChildrenContextsList(): ChildrenContextsPerParent; + addChildrenContexts(value?: ml_metadata_proto_metadata_store_pb.Context, index?: number): ml_metadata_proto_metadata_store_pb.Context; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): ChildrenContextsPerParent.AsObject; + static toObject(includeInstance: boolean, msg: ChildrenContextsPerParent): ChildrenContextsPerParent.AsObject; + static serializeBinaryToWriter(message: ChildrenContextsPerParent, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): ChildrenContextsPerParent; + static deserializeBinaryFromReader(message: ChildrenContextsPerParent, reader: jspb.BinaryReader): ChildrenContextsPerParent; + } + + export namespace ChildrenContextsPerParent { + export type AsObject = { + childrenContextsList: Array, + } + } + +} + export class GetArtifactsByContextRequest extends jspb.Message { getContextId(): number; setContextId(value: number): GetArtifactsByContextRequest; @@ -2211,3 +2850,55 @@ export namespace GetLineageGraphResponse { } } +export class GetLineageSubgraphRequest extends jspb.Message { + getLineageSubgraphQueryOptions(): ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions | undefined; + setLineageSubgraphQueryOptions(value?: ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions): GetLineageSubgraphRequest; + hasLineageSubgraphQueryOptions(): boolean; + clearLineageSubgraphQueryOptions(): GetLineageSubgraphRequest; + + getReadMask(): google_protobuf_field_mask_pb.FieldMask | undefined; + setReadMask(value?: google_protobuf_field_mask_pb.FieldMask): GetLineageSubgraphRequest; + hasReadMask(): boolean; + clearReadMask(): GetLineageSubgraphRequest; + + getTransactionOptions(): ml_metadata_proto_metadata_store_pb.TransactionOptions | undefined; + setTransactionOptions(value?: ml_metadata_proto_metadata_store_pb.TransactionOptions): GetLineageSubgraphRequest; + hasTransactionOptions(): boolean; + clearTransactionOptions(): GetLineageSubgraphRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetLineageSubgraphRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetLineageSubgraphRequest): GetLineageSubgraphRequest.AsObject; + static serializeBinaryToWriter(message: GetLineageSubgraphRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetLineageSubgraphRequest; + static deserializeBinaryFromReader(message: GetLineageSubgraphRequest, reader: jspb.BinaryReader): GetLineageSubgraphRequest; +} + +export namespace GetLineageSubgraphRequest { + export type AsObject = { + lineageSubgraphQueryOptions?: ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions.AsObject, + readMask?: google_protobuf_field_mask_pb.FieldMask.AsObject, + transactionOptions?: ml_metadata_proto_metadata_store_pb.TransactionOptions.AsObject, + } +} + +export class GetLineageSubgraphResponse extends jspb.Message { + getLineageSubgraph(): ml_metadata_proto_metadata_store_pb.LineageGraph | undefined; + setLineageSubgraph(value?: ml_metadata_proto_metadata_store_pb.LineageGraph): GetLineageSubgraphResponse; + hasLineageSubgraph(): boolean; + clearLineageSubgraph(): GetLineageSubgraphResponse; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetLineageSubgraphResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetLineageSubgraphResponse): GetLineageSubgraphResponse.AsObject; + static serializeBinaryToWriter(message: GetLineageSubgraphResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetLineageSubgraphResponse; + static deserializeBinaryFromReader(message: GetLineageSubgraphResponse, reader: jspb.BinaryReader): GetLineageSubgraphResponse; +} + +export namespace GetLineageSubgraphResponse { + export type AsObject = { + lineageSubgraph?: ml_metadata_proto_metadata_store_pb.LineageGraph.AsObject, + } +} + diff --git a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.js b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.js index c4fdbcaec2..79ddeebab6 100644 --- a/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.js +++ b/frontend/src/third_party/mlmd/generated/ml_metadata/proto/metadata_store_service_pb.js @@ -15,6 +15,8 @@ var jspb = require('google-protobuf'); var goog = jspb; var global = Function('return this')(); +var google_protobuf_field_mask_pb = require('google-protobuf/google/protobuf/field_mask_pb.js'); +goog.object.extend(proto, google_protobuf_field_mask_pb); var ml_metadata_proto_metadata_store_pb = require('../../ml_metadata/proto/metadata_store_pb.js'); goog.object.extend(proto, ml_metadata_proto_metadata_store_pb); goog.exportSymbol('proto.ml_metadata.ArtifactAndType', null, global); @@ -26,12 +28,16 @@ goog.exportSymbol('proto.ml_metadata.GetArtifactByTypeAndNameRequest', null, glo goog.exportSymbol('proto.ml_metadata.GetArtifactByTypeAndNameResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactTypeRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactTypeResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetArtifactTypesByExternalIdsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetArtifactTypesByExternalIdsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactTypesByIDRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactTypesByIDResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactTypesRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactTypesResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactsByContextRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactsByContextResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetArtifactsByExternalIdsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetArtifactsByExternalIdsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactsByIDRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactsByIDResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactsByTypeRequest', null, global); @@ -42,10 +48,15 @@ goog.exportSymbol('proto.ml_metadata.GetArtifactsRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetArtifactsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetChildrenContextsByContextRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetChildrenContextsByContextResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetChildrenContextsByContextsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetChildrenContextsByContextsResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent', null, global); goog.exportSymbol('proto.ml_metadata.GetContextByTypeAndNameRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetContextByTypeAndNameResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetContextTypeRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetContextTypeResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetContextTypesByExternalIdsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetContextTypesByExternalIdsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetContextTypesByIDRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetContextTypesByIDResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetContextTypesRequest', null, global); @@ -54,6 +65,8 @@ goog.exportSymbol('proto.ml_metadata.GetContextsByArtifactRequest', null, global goog.exportSymbol('proto.ml_metadata.GetContextsByArtifactResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetContextsByExecutionRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetContextsByExecutionResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetContextsByExternalIdsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetContextsByExternalIdsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetContextsByIDRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetContextsByIDResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetContextsByTypeRequest', null, global); @@ -68,12 +81,16 @@ goog.exportSymbol('proto.ml_metadata.GetExecutionByTypeAndNameRequest', null, gl goog.exportSymbol('proto.ml_metadata.GetExecutionByTypeAndNameResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionTypeRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionTypeResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetExecutionTypesByExternalIdsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetExecutionTypesByExternalIdsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionTypesByIDRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionTypesByIDResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionTypesRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionTypesResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionsByContextRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionsByContextResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetExecutionsByExternalIdsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetExecutionsByExternalIdsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionsByIDRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionsByIDResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionsByTypeRequest', null, global); @@ -82,8 +99,13 @@ goog.exportSymbol('proto.ml_metadata.GetExecutionsRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetExecutionsResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetLineageGraphRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetLineageGraphResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetLineageSubgraphRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetLineageSubgraphResponse', null, global); goog.exportSymbol('proto.ml_metadata.GetParentContextsByContextRequest', null, global); goog.exportSymbol('proto.ml_metadata.GetParentContextsByContextResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetParentContextsByContextsRequest', null, global); +goog.exportSymbol('proto.ml_metadata.GetParentContextsByContextsResponse', null, global); +goog.exportSymbol('proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild', null, global); goog.exportSymbol('proto.ml_metadata.PutArtifactTypeRequest', null, global); goog.exportSymbol('proto.ml_metadata.PutArtifactTypeResponse', null, global); goog.exportSymbol('proto.ml_metadata.PutArtifactsRequest', null, global); @@ -105,6 +127,10 @@ goog.exportSymbol('proto.ml_metadata.PutExecutionTypeRequest', null, global); goog.exportSymbol('proto.ml_metadata.PutExecutionTypeResponse', null, global); goog.exportSymbol('proto.ml_metadata.PutExecutionsRequest', null, global); goog.exportSymbol('proto.ml_metadata.PutExecutionsResponse', null, global); +goog.exportSymbol('proto.ml_metadata.PutLineageSubgraphRequest', null, global); +goog.exportSymbol('proto.ml_metadata.PutLineageSubgraphRequest.EventEdge', null, global); +goog.exportSymbol('proto.ml_metadata.PutLineageSubgraphRequest.Options', null, global); +goog.exportSymbol('proto.ml_metadata.PutLineageSubgraphResponse', null, global); goog.exportSymbol('proto.ml_metadata.PutParentContextsRequest', null, global); goog.exportSymbol('proto.ml_metadata.PutParentContextsResponse', null, global); goog.exportSymbol('proto.ml_metadata.PutTypesRequest', null, global); @@ -508,6 +534,90 @@ if (goog.DEBUG && !COMPILED) { */ proto.ml_metadata.PutExecutionResponse.displayName = 'proto.ml_metadata.PutExecutionResponse'; } +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.PutLineageSubgraphRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.PutLineageSubgraphRequest.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.PutLineageSubgraphRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.PutLineageSubgraphRequest.displayName = 'proto.ml_metadata.PutLineageSubgraphRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.PutLineageSubgraphRequest.EventEdge, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.displayName = 'proto.ml_metadata.PutLineageSubgraphRequest.EventEdge'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.PutLineageSubgraphRequest.Options, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.PutLineageSubgraphRequest.Options.displayName = 'proto.ml_metadata.PutLineageSubgraphRequest.Options'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.PutLineageSubgraphResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.PutLineageSubgraphResponse.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.PutLineageSubgraphResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.PutLineageSubgraphResponse.displayName = 'proto.ml_metadata.PutLineageSubgraphResponse'; +} /** * Generated by JsPbCodeGenerator. * @param {Array=} opt_data Optional initial data array, typically from a @@ -1148,16 +1258,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionsByTypeRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetArtifactsByExternalIdsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactsByExternalIdsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionsByTypeRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetArtifactsByExternalIdsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionsByTypeRequest.displayName = 'proto.ml_metadata.GetExecutionsByTypeRequest'; + proto.ml_metadata.GetArtifactsByExternalIdsRequest.displayName = 'proto.ml_metadata.GetArtifactsByExternalIdsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1169,16 +1279,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionsByTypeResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByTypeResponse.repeatedFields_, null); +proto.ml_metadata.GetArtifactsByExternalIdsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactsByExternalIdsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionsByTypeResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetArtifactsByExternalIdsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionsByTypeResponse.displayName = 'proto.ml_metadata.GetExecutionsByTypeResponse'; + proto.ml_metadata.GetArtifactsByExternalIdsResponse.displayName = 'proto.ml_metadata.GetArtifactsByExternalIdsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1190,16 +1300,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetExecutionsByExternalIdsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByExternalIdsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionByTypeAndNameRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionsByExternalIdsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionByTypeAndNameRequest.displayName = 'proto.ml_metadata.GetExecutionByTypeAndNameRequest'; + proto.ml_metadata.GetExecutionsByExternalIdsRequest.displayName = 'proto.ml_metadata.GetExecutionsByExternalIdsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1211,16 +1321,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetExecutionsByExternalIdsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByExternalIdsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionByTypeAndNameResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionsByExternalIdsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionByTypeAndNameResponse.displayName = 'proto.ml_metadata.GetExecutionByTypeAndNameResponse'; + proto.ml_metadata.GetExecutionsByExternalIdsResponse.displayName = 'proto.ml_metadata.GetExecutionsByExternalIdsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1232,16 +1342,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionsByIDRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByIDRequest.repeatedFields_, null); +proto.ml_metadata.GetContextsByExternalIdsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByExternalIdsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionsByIDRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByExternalIdsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionsByIDRequest.displayName = 'proto.ml_metadata.GetExecutionsByIDRequest'; + proto.ml_metadata.GetContextsByExternalIdsRequest.displayName = 'proto.ml_metadata.GetContextsByExternalIdsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1253,16 +1363,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionsByIDResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByIDResponse.repeatedFields_, null); +proto.ml_metadata.GetContextsByExternalIdsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByExternalIdsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionsByIDResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByExternalIdsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionsByIDResponse.displayName = 'proto.ml_metadata.GetExecutionsByIDResponse'; + proto.ml_metadata.GetContextsByExternalIdsResponse.displayName = 'proto.ml_metadata.GetContextsByExternalIdsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1274,16 +1384,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionTypeRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionTypeRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetArtifactTypesByExternalIdsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionTypeRequest.displayName = 'proto.ml_metadata.GetExecutionTypeRequest'; + proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.displayName = 'proto.ml_metadata.GetArtifactTypesByExternalIdsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1295,16 +1405,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionTypeResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionTypeResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetArtifactTypesByExternalIdsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionTypeResponse.displayName = 'proto.ml_metadata.GetExecutionTypeResponse'; + proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.displayName = 'proto.ml_metadata.GetArtifactTypesByExternalIdsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1316,16 +1426,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetEventsByExecutionIDsRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByExecutionIDsRequest.repeatedFields_, null); +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetEventsByExecutionIDsRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionTypesByExternalIdsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetEventsByExecutionIDsRequest.displayName = 'proto.ml_metadata.GetEventsByExecutionIDsRequest'; + proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.displayName = 'proto.ml_metadata.GetExecutionTypesByExternalIdsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1337,16 +1447,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetEventsByExecutionIDsResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByExecutionIDsResponse.repeatedFields_, null); +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetEventsByExecutionIDsResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionTypesByExternalIdsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetEventsByExecutionIDsResponse.displayName = 'proto.ml_metadata.GetEventsByExecutionIDsResponse'; + proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.displayName = 'proto.ml_metadata.GetExecutionTypesByExternalIdsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1358,16 +1468,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetEventsByArtifactIDsRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByArtifactIDsRequest.repeatedFields_, null); +proto.ml_metadata.GetContextTypesByExternalIdsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextTypesByExternalIdsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetEventsByArtifactIDsRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextTypesByExternalIdsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetEventsByArtifactIDsRequest.displayName = 'proto.ml_metadata.GetEventsByArtifactIDsRequest'; + proto.ml_metadata.GetContextTypesByExternalIdsRequest.displayName = 'proto.ml_metadata.GetContextTypesByExternalIdsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1379,16 +1489,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetEventsByArtifactIDsResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByArtifactIDsResponse.repeatedFields_, null); +proto.ml_metadata.GetContextTypesByExternalIdsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextTypesByExternalIdsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetEventsByArtifactIDsResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextTypesByExternalIdsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetEventsByArtifactIDsResponse.displayName = 'proto.ml_metadata.GetEventsByArtifactIDsResponse'; + proto.ml_metadata.GetContextTypesByExternalIdsResponse.displayName = 'proto.ml_metadata.GetContextTypesByExternalIdsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1400,16 +1510,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetArtifactTypesByIDRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactTypesByIDRequest.repeatedFields_, null); +proto.ml_metadata.GetExecutionsByTypeRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetArtifactTypesByIDRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionsByTypeRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetArtifactTypesByIDRequest.displayName = 'proto.ml_metadata.GetArtifactTypesByIDRequest'; + proto.ml_metadata.GetExecutionsByTypeRequest.displayName = 'proto.ml_metadata.GetExecutionsByTypeRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1421,16 +1531,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetArtifactTypesByIDResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactTypesByIDResponse.repeatedFields_, null); +proto.ml_metadata.GetExecutionsByTypeResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByTypeResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetArtifactTypesByIDResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionsByTypeResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetArtifactTypesByIDResponse.displayName = 'proto.ml_metadata.GetArtifactTypesByIDResponse'; + proto.ml_metadata.GetExecutionsByTypeResponse.displayName = 'proto.ml_metadata.GetExecutionsByTypeResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1442,16 +1552,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionTypesByIDRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionTypesByIDRequest.repeatedFields_, null); +proto.ml_metadata.GetExecutionByTypeAndNameRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetExecutionTypesByIDRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionByTypeAndNameRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionTypesByIDRequest.displayName = 'proto.ml_metadata.GetExecutionTypesByIDRequest'; + proto.ml_metadata.GetExecutionByTypeAndNameRequest.displayName = 'proto.ml_metadata.GetExecutionByTypeAndNameRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1463,16 +1573,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionTypesByIDResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionTypesByIDResponse.repeatedFields_, null); +proto.ml_metadata.GetExecutionByTypeAndNameResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetExecutionTypesByIDResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionByTypeAndNameResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionTypesByIDResponse.displayName = 'proto.ml_metadata.GetExecutionTypesByIDResponse'; + proto.ml_metadata.GetExecutionByTypeAndNameResponse.displayName = 'proto.ml_metadata.GetExecutionByTypeAndNameResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1484,16 +1594,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextTypeRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetExecutionsByIDRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByIDRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextTypeRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionsByIDRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextTypeRequest.displayName = 'proto.ml_metadata.GetContextTypeRequest'; + proto.ml_metadata.GetExecutionsByIDRequest.displayName = 'proto.ml_metadata.GetExecutionsByIDRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1505,16 +1615,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextTypeResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetExecutionsByIDResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByIDResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextTypeResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionsByIDResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextTypeResponse.displayName = 'proto.ml_metadata.GetContextTypeResponse'; + proto.ml_metadata.GetExecutionsByIDResponse.displayName = 'proto.ml_metadata.GetExecutionsByIDResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1526,16 +1636,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextTypesByIDRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextTypesByIDRequest.repeatedFields_, null); +proto.ml_metadata.GetExecutionTypeRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetContextTypesByIDRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionTypeRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextTypesByIDRequest.displayName = 'proto.ml_metadata.GetContextTypesByIDRequest'; + proto.ml_metadata.GetExecutionTypeRequest.displayName = 'proto.ml_metadata.GetExecutionTypeRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1547,16 +1657,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextTypesByIDResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextTypesByIDResponse.repeatedFields_, null); +proto.ml_metadata.GetExecutionTypeResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetContextTypesByIDResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionTypeResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextTypesByIDResponse.displayName = 'proto.ml_metadata.GetContextTypesByIDResponse'; + proto.ml_metadata.GetExecutionTypeResponse.displayName = 'proto.ml_metadata.GetExecutionTypeResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1568,16 +1678,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetEventsByExecutionIDsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByExecutionIDsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetEventsByExecutionIDsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsRequest.displayName = 'proto.ml_metadata.GetContextsRequest'; + proto.ml_metadata.GetEventsByExecutionIDsRequest.displayName = 'proto.ml_metadata.GetEventsByExecutionIDsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1589,16 +1699,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsResponse.repeatedFields_, null); +proto.ml_metadata.GetEventsByExecutionIDsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByExecutionIDsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetEventsByExecutionIDsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsResponse.displayName = 'proto.ml_metadata.GetContextsResponse'; + proto.ml_metadata.GetEventsByExecutionIDsResponse.displayName = 'proto.ml_metadata.GetEventsByExecutionIDsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1610,16 +1720,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByTypeRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetEventsByArtifactIDsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByArtifactIDsRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsByTypeRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetEventsByArtifactIDsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByTypeRequest.displayName = 'proto.ml_metadata.GetContextsByTypeRequest'; + proto.ml_metadata.GetEventsByArtifactIDsRequest.displayName = 'proto.ml_metadata.GetEventsByArtifactIDsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1631,16 +1741,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByTypeResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByTypeResponse.repeatedFields_, null); +proto.ml_metadata.GetEventsByArtifactIDsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetEventsByArtifactIDsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsByTypeResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetEventsByArtifactIDsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByTypeResponse.displayName = 'proto.ml_metadata.GetContextsByTypeResponse'; + proto.ml_metadata.GetEventsByArtifactIDsResponse.displayName = 'proto.ml_metadata.GetEventsByArtifactIDsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1652,16 +1762,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextByTypeAndNameRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetArtifactTypesByIDRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactTypesByIDRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextByTypeAndNameRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetArtifactTypesByIDRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextByTypeAndNameRequest.displayName = 'proto.ml_metadata.GetContextByTypeAndNameRequest'; + proto.ml_metadata.GetArtifactTypesByIDRequest.displayName = 'proto.ml_metadata.GetArtifactTypesByIDRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1673,16 +1783,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextByTypeAndNameResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetArtifactTypesByIDResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactTypesByIDResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextByTypeAndNameResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetArtifactTypesByIDResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextByTypeAndNameResponse.displayName = 'proto.ml_metadata.GetContextByTypeAndNameResponse'; + proto.ml_metadata.GetArtifactTypesByIDResponse.displayName = 'proto.ml_metadata.GetArtifactTypesByIDResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1694,16 +1804,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByIDRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByIDRequest.repeatedFields_, null); +proto.ml_metadata.GetExecutionTypesByIDRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionTypesByIDRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsByIDRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionTypesByIDRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByIDRequest.displayName = 'proto.ml_metadata.GetContextsByIDRequest'; + proto.ml_metadata.GetExecutionTypesByIDRequest.displayName = 'proto.ml_metadata.GetExecutionTypesByIDRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1715,16 +1825,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByIDResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByIDResponse.repeatedFields_, null); +proto.ml_metadata.GetExecutionTypesByIDResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionTypesByIDResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsByIDResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetExecutionTypesByIDResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByIDResponse.displayName = 'proto.ml_metadata.GetContextsByIDResponse'; + proto.ml_metadata.GetExecutionTypesByIDResponse.displayName = 'proto.ml_metadata.GetExecutionTypesByIDResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1736,16 +1846,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByArtifactRequest = function(opt_data) { +proto.ml_metadata.GetContextTypeRequest = function(opt_data) { jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetContextsByArtifactRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextTypeRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByArtifactRequest.displayName = 'proto.ml_metadata.GetContextsByArtifactRequest'; + proto.ml_metadata.GetContextTypeRequest.displayName = 'proto.ml_metadata.GetContextTypeRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1757,16 +1867,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByArtifactResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByArtifactResponse.repeatedFields_, null); +proto.ml_metadata.GetContextTypeResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetContextsByArtifactResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextTypeResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByArtifactResponse.displayName = 'proto.ml_metadata.GetContextsByArtifactResponse'; + proto.ml_metadata.GetContextTypeResponse.displayName = 'proto.ml_metadata.GetContextTypeResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1778,16 +1888,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByExecutionRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetContextTypesByIDRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextTypesByIDRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsByExecutionRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextTypesByIDRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByExecutionRequest.displayName = 'proto.ml_metadata.GetContextsByExecutionRequest'; + proto.ml_metadata.GetContextTypesByIDRequest.displayName = 'proto.ml_metadata.GetContextTypesByIDRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1799,16 +1909,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetContextsByExecutionResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByExecutionResponse.repeatedFields_, null); +proto.ml_metadata.GetContextTypesByIDResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextTypesByIDResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetContextsByExecutionResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextTypesByIDResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetContextsByExecutionResponse.displayName = 'proto.ml_metadata.GetContextsByExecutionResponse'; + proto.ml_metadata.GetContextTypesByIDResponse.displayName = 'proto.ml_metadata.GetContextTypesByIDResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1820,16 +1930,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetParentContextsByContextRequest = function(opt_data) { +proto.ml_metadata.GetContextsRequest = function(opt_data) { jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetParentContextsByContextRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetParentContextsByContextRequest.displayName = 'proto.ml_metadata.GetParentContextsByContextRequest'; + proto.ml_metadata.GetContextsRequest.displayName = 'proto.ml_metadata.GetContextsRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1841,16 +1951,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetParentContextsByContextResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetParentContextsByContextResponse.repeatedFields_, null); +proto.ml_metadata.GetContextsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetParentContextsByContextResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetParentContextsByContextResponse.displayName = 'proto.ml_metadata.GetParentContextsByContextResponse'; + proto.ml_metadata.GetContextsResponse.displayName = 'proto.ml_metadata.GetContextsResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1862,16 +1972,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetChildrenContextsByContextRequest = function(opt_data) { +proto.ml_metadata.GetContextsByTypeRequest = function(opt_data) { jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetChildrenContextsByContextRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByTypeRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetChildrenContextsByContextRequest.displayName = 'proto.ml_metadata.GetChildrenContextsByContextRequest'; + proto.ml_metadata.GetContextsByTypeRequest.displayName = 'proto.ml_metadata.GetContextsByTypeRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1883,16 +1993,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetChildrenContextsByContextResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetChildrenContextsByContextResponse.repeatedFields_, null); +proto.ml_metadata.GetContextsByTypeResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByTypeResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetChildrenContextsByContextResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByTypeResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetChildrenContextsByContextResponse.displayName = 'proto.ml_metadata.GetChildrenContextsByContextResponse'; + proto.ml_metadata.GetContextsByTypeResponse.displayName = 'proto.ml_metadata.GetContextsByTypeResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1904,16 +2014,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetArtifactsByContextRequest = function(opt_data) { +proto.ml_metadata.GetContextByTypeAndNameRequest = function(opt_data) { jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetArtifactsByContextRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextByTypeAndNameRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetArtifactsByContextRequest.displayName = 'proto.ml_metadata.GetArtifactsByContextRequest'; + proto.ml_metadata.GetContextByTypeAndNameRequest.displayName = 'proto.ml_metadata.GetContextByTypeAndNameRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1925,16 +2035,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetArtifactsByContextResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactsByContextResponse.repeatedFields_, null); +proto.ml_metadata.GetContextByTypeAndNameResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetArtifactsByContextResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextByTypeAndNameResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetArtifactsByContextResponse.displayName = 'proto.ml_metadata.GetArtifactsByContextResponse'; + proto.ml_metadata.GetContextByTypeAndNameResponse.displayName = 'proto.ml_metadata.GetContextByTypeAndNameResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1946,16 +2056,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionsByContextRequest = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetContextsByIDRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByIDRequest.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionsByContextRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByIDRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionsByContextRequest.displayName = 'proto.ml_metadata.GetExecutionsByContextRequest'; + proto.ml_metadata.GetContextsByIDRequest.displayName = 'proto.ml_metadata.GetContextsByIDRequest'; } /** * Generated by JsPbCodeGenerator. @@ -1967,16 +2077,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetExecutionsByContextResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByContextResponse.repeatedFields_, null); +proto.ml_metadata.GetContextsByIDResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByIDResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetExecutionsByContextResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByIDResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetExecutionsByContextResponse.displayName = 'proto.ml_metadata.GetExecutionsByContextResponse'; + proto.ml_metadata.GetContextsByIDResponse.displayName = 'proto.ml_metadata.GetContextsByIDResponse'; } /** * Generated by JsPbCodeGenerator. @@ -1988,16 +2098,16 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetLineageGraphRequest = function(opt_data) { +proto.ml_metadata.GetContextsByArtifactRequest = function(opt_data) { jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; -goog.inherits(proto.ml_metadata.GetLineageGraphRequest, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByArtifactRequest, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetLineageGraphRequest.displayName = 'proto.ml_metadata.GetLineageGraphRequest'; + proto.ml_metadata.GetContextsByArtifactRequest.displayName = 'proto.ml_metadata.GetContextsByArtifactRequest'; } /** * Generated by JsPbCodeGenerator. @@ -2009,170 +2119,6762 @@ if (goog.DEBUG && !COMPILED) { * @extends {jspb.Message} * @constructor */ -proto.ml_metadata.GetLineageGraphResponse = function(opt_data) { - jspb.Message.initialize(this, opt_data, 0, -1, null, null); +proto.ml_metadata.GetContextsByArtifactResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByArtifactResponse.repeatedFields_, null); }; -goog.inherits(proto.ml_metadata.GetLineageGraphResponse, jspb.Message); +goog.inherits(proto.ml_metadata.GetContextsByArtifactResponse, jspb.Message); if (goog.DEBUG && !COMPILED) { /** * @public * @override */ - proto.ml_metadata.GetLineageGraphResponse.displayName = 'proto.ml_metadata.GetLineageGraphResponse'; + proto.ml_metadata.GetContextsByArtifactResponse.displayName = 'proto.ml_metadata.GetContextsByArtifactResponse'; } - - - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor */ -proto.ml_metadata.ArtifactAndType.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ArtifactAndType.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByExecutionRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; - - +goog.inherits(proto.ml_metadata.GetContextsByExecutionRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetContextsByExecutionRequest.displayName = 'proto.ml_metadata.GetContextsByExecutionRequest'; +} /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ArtifactAndType} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor */ -proto.ml_metadata.ArtifactAndType.toObject = function(includeInstance, msg) { - var f, obj = { - artifact: (f = msg.getArtifact()) && ml_metadata_proto_metadata_store_pb.Artifact.toObject(includeInstance, f), - type: (f = msg.getType()) && ml_metadata_proto_metadata_store_pb.ArtifactType.toObject(includeInstance, f) - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +proto.ml_metadata.GetContextsByExecutionResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetContextsByExecutionResponse.repeatedFields_, null); }; +goog.inherits(proto.ml_metadata.GetContextsByExecutionResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetContextsByExecutionResponse.displayName = 'proto.ml_metadata.GetContextsByExecutionResponse'; } - - /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ArtifactAndType} + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor */ -proto.ml_metadata.ArtifactAndType.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ArtifactAndType; - return proto.ml_metadata.ArtifactAndType.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.GetParentContextsByContextRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); }; - - +goog.inherits(proto.ml_metadata.GetParentContextsByContextRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetParentContextsByContextRequest.displayName = 'proto.ml_metadata.GetParentContextsByContextRequest'; +} /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.ArtifactAndType} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ArtifactAndType} + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor */ -proto.ml_metadata.ArtifactAndType.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.setArtifact(value); - break; - case 2: - var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); - msg.setType(value); - break; - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.GetParentContextsByContextResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetParentContextsByContextResponse.repeatedFields_, null); }; - - +goog.inherits(proto.ml_metadata.GetParentContextsByContextResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetParentContextsByContextResponse.displayName = 'proto.ml_metadata.GetParentContextsByContextResponse'; +} /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor */ -proto.ml_metadata.ArtifactAndType.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ArtifactAndType.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.GetChildrenContextsByContextRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetChildrenContextsByContextRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetChildrenContextsByContextRequest.displayName = 'proto.ml_metadata.GetChildrenContextsByContextRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetChildrenContextsByContextResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetChildrenContextsByContextResponse.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.GetChildrenContextsByContextResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetChildrenContextsByContextResponse.displayName = 'proto.ml_metadata.GetChildrenContextsByContextResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetParentContextsByContextsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetParentContextsByContextsRequest.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.GetParentContextsByContextsRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetParentContextsByContextsRequest.displayName = 'proto.ml_metadata.GetParentContextsByContextsRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetParentContextsByContextsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetParentContextsByContextsResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetParentContextsByContextsResponse.displayName = 'proto.ml_metadata.GetParentContextsByContextsResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.displayName = 'proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetChildrenContextsByContextsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetChildrenContextsByContextsRequest.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.GetChildrenContextsByContextsRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetChildrenContextsByContextsRequest.displayName = 'proto.ml_metadata.GetChildrenContextsByContextsRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetChildrenContextsByContextsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetChildrenContextsByContextsResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetChildrenContextsByContextsResponse.displayName = 'proto.ml_metadata.GetChildrenContextsByContextsResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.displayName = 'proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetArtifactsByContextRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetArtifactsByContextRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetArtifactsByContextRequest.displayName = 'proto.ml_metadata.GetArtifactsByContextRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetArtifactsByContextResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetArtifactsByContextResponse.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.GetArtifactsByContextResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetArtifactsByContextResponse.displayName = 'proto.ml_metadata.GetArtifactsByContextResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetExecutionsByContextRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetExecutionsByContextRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetExecutionsByContextRequest.displayName = 'proto.ml_metadata.GetExecutionsByContextRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetExecutionsByContextResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.ml_metadata.GetExecutionsByContextResponse.repeatedFields_, null); +}; +goog.inherits(proto.ml_metadata.GetExecutionsByContextResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetExecutionsByContextResponse.displayName = 'proto.ml_metadata.GetExecutionsByContextResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetLineageGraphRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetLineageGraphRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetLineageGraphRequest.displayName = 'proto.ml_metadata.GetLineageGraphRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetLineageGraphResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetLineageGraphResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetLineageGraphResponse.displayName = 'proto.ml_metadata.GetLineageGraphResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetLineageSubgraphRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetLineageSubgraphRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetLineageSubgraphRequest.displayName = 'proto.ml_metadata.GetLineageSubgraphRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.ml_metadata.GetLineageSubgraphResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.ml_metadata.GetLineageSubgraphResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.ml_metadata.GetLineageSubgraphResponse.displayName = 'proto.ml_metadata.GetLineageSubgraphResponse'; +} + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.ArtifactAndType.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ArtifactAndType.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.ArtifactAndType} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactAndType.toObject = function(includeInstance, msg) { + var f, obj = { + artifact: (f = msg.getArtifact()) && ml_metadata_proto_metadata_store_pb.Artifact.toObject(includeInstance, f), + type: (f = msg.getType()) && ml_metadata_proto_metadata_store_pb.ArtifactType.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.ArtifactAndType} + */ +proto.ml_metadata.ArtifactAndType.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.ArtifactAndType; + return proto.ml_metadata.ArtifactAndType.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.ArtifactAndType} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.ArtifactAndType} + */ +proto.ml_metadata.ArtifactAndType.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.setArtifact(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.setType(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.ArtifactAndType.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.ArtifactAndType.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.ArtifactAndType} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactAndType.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifact(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } + f = message.getType(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ); + } +}; + + +/** + * optional Artifact artifact = 1; + * @return {?proto.ml_metadata.Artifact} + */ +proto.ml_metadata.ArtifactAndType.prototype.getArtifact = function() { + return /** @type{?proto.ml_metadata.Artifact} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +}; + + +/** + * @param {?proto.ml_metadata.Artifact|undefined} value + * @return {!proto.ml_metadata.ArtifactAndType} returns this +*/ +proto.ml_metadata.ArtifactAndType.prototype.setArtifact = function(value) { + return jspb.Message.setWrapperField(this, 1, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ArtifactAndType} returns this + */ +proto.ml_metadata.ArtifactAndType.prototype.clearArtifact = function() { + return this.setArtifact(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ArtifactAndType.prototype.hasArtifact = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional ArtifactType type = 2; + * @return {?proto.ml_metadata.ArtifactType} + */ +proto.ml_metadata.ArtifactAndType.prototype.getType = function() { + return /** @type{?proto.ml_metadata.ArtifactType} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 2)); +}; + + +/** + * @param {?proto.ml_metadata.ArtifactType|undefined} value + * @return {!proto.ml_metadata.ArtifactAndType} returns this +*/ +proto.ml_metadata.ArtifactAndType.prototype.setType = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ArtifactAndType} returns this + */ +proto.ml_metadata.ArtifactAndType.prototype.clearType = function() { + return this.setType(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ArtifactAndType.prototype.hasType = function() { + return jspb.Message.getField(this, 2) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.ArtifactStructMap.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ArtifactStructMap.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.ArtifactStructMap} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactStructMap.toObject = function(includeInstance, msg) { + var f, obj = { + propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.ArtifactStruct.toObject) : [] + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.ArtifactStructMap} + */ +proto.ml_metadata.ArtifactStructMap.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.ArtifactStructMap; + return proto.ml_metadata.ArtifactStructMap.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.ArtifactStructMap} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.ArtifactStructMap} + */ +proto.ml_metadata.ArtifactStructMap.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = msg.getPropertiesMap(); + reader.readMessage(value, function(message, reader) { + jspb.Map.deserializeBinary(message, reader, jspb.BinaryReader.prototype.readString, jspb.BinaryReader.prototype.readMessage, proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader, "", new proto.ml_metadata.ArtifactStruct()); + }); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.ArtifactStructMap.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.ArtifactStructMap.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.ArtifactStructMap} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactStructMap.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getPropertiesMap(true); + if (f && f.getLength() > 0) { + f.serializeBinary(1, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeMessage, proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter); + } +}; + + +/** + * map properties = 1; + * @param {boolean=} opt_noLazyCreate Do not create the map if + * empty, instead returning `undefined` + * @return {!jspb.Map} + */ +proto.ml_metadata.ArtifactStructMap.prototype.getPropertiesMap = function(opt_noLazyCreate) { + return /** @type {!jspb.Map} */ ( + jspb.Message.getMapField(this, 1, opt_noLazyCreate, + proto.ml_metadata.ArtifactStruct)); +}; + + +/** + * Clears values from the map. The map will be non-null. + * @return {!proto.ml_metadata.ArtifactStructMap} returns this + */ +proto.ml_metadata.ArtifactStructMap.prototype.clearPropertiesMap = function() { + this.getPropertiesMap().clear(); + return this;}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.ArtifactStructList.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.ArtifactStructList.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ArtifactStructList.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.ArtifactStructList} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactStructList.toObject = function(includeInstance, msg) { + var f, obj = { + elementsList: jspb.Message.toObjectList(msg.getElementsList(), + proto.ml_metadata.ArtifactStruct.toObject, includeInstance) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.ArtifactStructList} + */ +proto.ml_metadata.ArtifactStructList.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.ArtifactStructList; + return proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.ArtifactStructList} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.ArtifactStructList} + */ +proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.ml_metadata.ArtifactStruct; + reader.readMessage(value,proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader); + msg.addElements(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.ArtifactStructList.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.ArtifactStructList.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.ArtifactStructList} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactStructList.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getElementsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated ArtifactStruct elements = 1; + * @return {!Array} + */ +proto.ml_metadata.ArtifactStructList.prototype.getElementsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.ml_metadata.ArtifactStruct, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.ArtifactStructList} returns this +*/ +proto.ml_metadata.ArtifactStructList.prototype.setElementsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.ArtifactStruct=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ArtifactStruct} + */ +proto.ml_metadata.ArtifactStructList.prototype.addElements = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactStruct, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.ArtifactStructList} returns this + */ +proto.ml_metadata.ArtifactStructList.prototype.clearElementsList = function() { + return this.setElementsList([]); +}; + + + +/** + * Oneof group definitions for this message. Each group defines the field + * numbers belonging to that group. When of these fields' value is set, all + * other fields in the group are cleared. During deserialization, if multiple + * fields are encountered for a group, only the last value seen will be kept. + * @private {!Array>} + * @const + */ +proto.ml_metadata.ArtifactStruct.oneofGroups_ = [[1,2,3]]; + +/** + * @enum {number} + */ +proto.ml_metadata.ArtifactStruct.ValueCase = { + VALUE_NOT_SET: 0, + ARTIFACT: 1, + MAP: 2, + LIST: 3 +}; + +/** + * @return {proto.ml_metadata.ArtifactStruct.ValueCase} + */ +proto.ml_metadata.ArtifactStruct.prototype.getValueCase = function() { + return /** @type {proto.ml_metadata.ArtifactStruct.ValueCase} */(jspb.Message.computeOneofCase(this, proto.ml_metadata.ArtifactStruct.oneofGroups_[0])); +}; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.ArtifactStruct.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.ArtifactStruct.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.ArtifactStruct} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactStruct.toObject = function(includeInstance, msg) { + var f, obj = { + artifact: (f = msg.getArtifact()) && proto.ml_metadata.ArtifactAndType.toObject(includeInstance, f), + map: (f = msg.getMap()) && proto.ml_metadata.ArtifactStructMap.toObject(includeInstance, f), + list: (f = msg.getList()) && proto.ml_metadata.ArtifactStructList.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.ArtifactStruct} + */ +proto.ml_metadata.ArtifactStruct.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.ArtifactStruct; + return proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.ArtifactStruct} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.ArtifactStruct} + */ +proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.ml_metadata.ArtifactAndType; + reader.readMessage(value,proto.ml_metadata.ArtifactAndType.deserializeBinaryFromReader); + msg.setArtifact(value); + break; + case 2: + var value = new proto.ml_metadata.ArtifactStructMap; + reader.readMessage(value,proto.ml_metadata.ArtifactStructMap.deserializeBinaryFromReader); + msg.setMap(value); + break; + case 3: + var value = new proto.ml_metadata.ArtifactStructList; + reader.readMessage(value,proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader); + msg.setList(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.ArtifactStruct.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.ArtifactStruct} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifact(); + if (f != null) { + writer.writeMessage( + 1, + f, + proto.ml_metadata.ArtifactAndType.serializeBinaryToWriter + ); + } + f = message.getMap(); + if (f != null) { + writer.writeMessage( + 2, + f, + proto.ml_metadata.ArtifactStructMap.serializeBinaryToWriter + ); + } + f = message.getList(); + if (f != null) { + writer.writeMessage( + 3, + f, + proto.ml_metadata.ArtifactStructList.serializeBinaryToWriter + ); + } +}; + + +/** + * optional ArtifactAndType artifact = 1; + * @return {?proto.ml_metadata.ArtifactAndType} + */ +proto.ml_metadata.ArtifactStruct.prototype.getArtifact = function() { + return /** @type{?proto.ml_metadata.ArtifactAndType} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.ArtifactAndType, 1)); +}; + + +/** + * @param {?proto.ml_metadata.ArtifactAndType|undefined} value + * @return {!proto.ml_metadata.ArtifactStruct} returns this +*/ +proto.ml_metadata.ArtifactStruct.prototype.setArtifact = function(value) { + return jspb.Message.setOneofWrapperField(this, 1, proto.ml_metadata.ArtifactStruct.oneofGroups_[0], value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ArtifactStruct} returns this + */ +proto.ml_metadata.ArtifactStruct.prototype.clearArtifact = function() { + return this.setArtifact(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ArtifactStruct.prototype.hasArtifact = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional ArtifactStructMap map = 2; + * @return {?proto.ml_metadata.ArtifactStructMap} + */ +proto.ml_metadata.ArtifactStruct.prototype.getMap = function() { + return /** @type{?proto.ml_metadata.ArtifactStructMap} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.ArtifactStructMap, 2)); +}; + + +/** + * @param {?proto.ml_metadata.ArtifactStructMap|undefined} value + * @return {!proto.ml_metadata.ArtifactStruct} returns this +*/ +proto.ml_metadata.ArtifactStruct.prototype.setMap = function(value) { + return jspb.Message.setOneofWrapperField(this, 2, proto.ml_metadata.ArtifactStruct.oneofGroups_[0], value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ArtifactStruct} returns this + */ +proto.ml_metadata.ArtifactStruct.prototype.clearMap = function() { + return this.setMap(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ArtifactStruct.prototype.hasMap = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional ArtifactStructList list = 3; + * @return {?proto.ml_metadata.ArtifactStructList} + */ +proto.ml_metadata.ArtifactStruct.prototype.getList = function() { + return /** @type{?proto.ml_metadata.ArtifactStructList} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.ArtifactStructList, 3)); +}; + + +/** + * @param {?proto.ml_metadata.ArtifactStructList|undefined} value + * @return {!proto.ml_metadata.ArtifactStruct} returns this +*/ +proto.ml_metadata.ArtifactStruct.prototype.setList = function(value) { + return jspb.Message.setOneofWrapperField(this, 3, proto.ml_metadata.ArtifactStruct.oneofGroups_[0], value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.ArtifactStruct} returns this + */ +proto.ml_metadata.ArtifactStruct.prototype.clearList = function() { + return this.setList(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.ArtifactStruct.prototype.hasList = function() { + return jspb.Message.getField(this, 3) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutArtifactsRequest.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutArtifactsRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutArtifactsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactsRequest.toObject = function(includeInstance, msg) { + var f, obj = { + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), + options: (f = msg.getOptions()) && proto.ml_metadata.PutArtifactsRequest.Options.toObject(includeInstance, f), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f), + updateMask: (f = msg.getUpdateMask()) && google_protobuf_field_mask_pb.FieldMask.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutArtifactsRequest} + */ +proto.ml_metadata.PutArtifactsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutArtifactsRequest; + return proto.ml_metadata.PutArtifactsRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutArtifactsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutArtifactsRequest} + */ +proto.ml_metadata.PutArtifactsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); + break; + case 2: + var value = new proto.ml_metadata.PutArtifactsRequest.Options; + reader.readMessage(value,proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader); + msg.setOptions(value); + break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + case 4: + var value = new google_protobuf_field_mask_pb.FieldMask; + reader.readMessage(value,google_protobuf_field_mask_pb.FieldMask.deserializeBinaryFromReader); + msg.setUpdateMask(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutArtifactsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutArtifactsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } + f = message.getOptions(); + if (f != null) { + writer.writeMessage( + 2, + f, + proto.ml_metadata.PutArtifactsRequest.Options.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } + f = message.getUpdateMask(); + if (f != null) { + writer.writeMessage( + 4, + f, + google_protobuf_field_mask_pb.FieldMask.serializeBinaryToWriter + ); + } +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutArtifactsRequest.Options.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutArtifactsRequest.Options.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutArtifactsRequest.Options} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactsRequest.Options.toObject = function(includeInstance, msg) { + var f, obj = { + abortIfLatestUpdatedTimeChanged: (f = jspb.Message.getBooleanField(msg, 1)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutArtifactsRequest.Options} + */ +proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutArtifactsRequest.Options; + return proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutArtifactsRequest.Options} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutArtifactsRequest.Options} + */ +proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setAbortIfLatestUpdatedTimeChanged(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutArtifactsRequest.Options.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutArtifactsRequest.Options.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutArtifactsRequest.Options} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactsRequest.Options.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {boolean} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeBool( + 1, + f + ); + } +}; + + +/** + * optional bool abort_if_latest_updated_time_changed = 1; + * @return {boolean} + */ +proto.ml_metadata.PutArtifactsRequest.Options.prototype.getAbortIfLatestUpdatedTimeChanged = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 1, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutArtifactsRequest.Options} returns this + */ +proto.ml_metadata.PutArtifactsRequest.Options.prototype.setAbortIfLatestUpdatedTimeChanged = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutArtifactsRequest.Options} returns this + */ +proto.ml_metadata.PutArtifactsRequest.Options.prototype.clearAbortIfLatestUpdatedTimeChanged = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactsRequest.Options.prototype.hasAbortIfLatestUpdatedTimeChanged = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * repeated Artifact artifacts = 1; + * @return {!Array} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this +*/ +proto.ml_metadata.PutArtifactsRequest.prototype.setArtifactsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Artifact} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + */ +proto.ml_metadata.PutArtifactsRequest.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); +}; + + +/** + * optional Options options = 2; + * @return {?proto.ml_metadata.PutArtifactsRequest.Options} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.PutArtifactsRequest.Options} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.PutArtifactsRequest.Options, 2)); +}; + + +/** + * @param {?proto.ml_metadata.PutArtifactsRequest.Options|undefined} value + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this +*/ +proto.ml_metadata.PutArtifactsRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + */ +proto.ml_metadata.PutArtifactsRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 3; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this +*/ +proto.ml_metadata.PutArtifactsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + */ +proto.ml_metadata.PutArtifactsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional google.protobuf.FieldMask update_mask = 4; + * @return {?proto.google.protobuf.FieldMask} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.getUpdateMask = function() { + return /** @type{?proto.google.protobuf.FieldMask} */ ( + jspb.Message.getWrapperField(this, google_protobuf_field_mask_pb.FieldMask, 4)); +}; + + +/** + * @param {?proto.google.protobuf.FieldMask|undefined} value + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this +*/ +proto.ml_metadata.PutArtifactsRequest.prototype.setUpdateMask = function(value) { + return jspb.Message.setWrapperField(this, 4, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + */ +proto.ml_metadata.PutArtifactsRequest.prototype.clearUpdateMask = function() { + return this.setUpdateMask(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactsRequest.prototype.hasUpdateMask = function() { + return jspb.Message.getField(this, 4) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutArtifactsResponse.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutArtifactsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutArtifactsResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutArtifactsResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactsResponse.toObject = function(includeInstance, msg) { + var f, obj = { + artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutArtifactsResponse} + */ +proto.ml_metadata.PutArtifactsResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutArtifactsResponse; + return proto.ml_metadata.PutArtifactsResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutArtifactsResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutArtifactsResponse} + */ +proto.ml_metadata.PutArtifactsResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addArtifactIds(values[i]); + } + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutArtifactsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutArtifactsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutArtifactsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 1, + f + ); + } +}; + + +/** + * repeated int64 artifact_ids = 1; + * @return {!Array} + */ +proto.ml_metadata.PutArtifactsResponse.prototype.getArtifactIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutArtifactsResponse} returns this + */ +proto.ml_metadata.PutArtifactsResponse.prototype.setArtifactIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutArtifactsResponse} returns this + */ +proto.ml_metadata.PutArtifactsResponse.prototype.addArtifactIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutArtifactsResponse} returns this + */ +proto.ml_metadata.PutArtifactsResponse.prototype.clearArtifactIdsList = function() { + return this.setArtifactIdsList([]); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutArtifactTypeRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutArtifactTypeRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactTypeRequest.toObject = function(includeInstance, msg) { + var f, obj = { + artifactType: (f = msg.getArtifactType()) && ml_metadata_proto_metadata_store_pb.ArtifactType.toObject(includeInstance, f), + canAddFields: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f, + canOmitFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, + canDeleteFields: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, + allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 4, true), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} + */ +proto.ml_metadata.PutArtifactTypeRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutArtifactTypeRequest; + return proto.ml_metadata.PutArtifactTypeRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutArtifactTypeRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} + */ +proto.ml_metadata.PutArtifactTypeRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.setArtifactType(value); + break; + case 2: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanAddFields(value); + break; + case 5: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanOmitFields(value); + break; + case 3: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanDeleteFields(value); + break; + case 4: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setAllFieldsMatch(value); + break; + case 6: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutArtifactTypeRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutArtifactTypeRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactTypeRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactType(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeBool( + 2, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeBool( + 5, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeBool( + 3, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeBool( + 4, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 6, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + +/** + * optional ArtifactType artifact_type = 1; + * @return {?proto.ml_metadata.ArtifactType} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.getArtifactType = function() { + return /** @type{?proto.ml_metadata.ArtifactType} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); +}; + + +/** + * @param {?proto.ml_metadata.ArtifactType|undefined} value + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this +*/ +proto.ml_metadata.PutArtifactTypeRequest.prototype.setArtifactType = function(value) { + return jspb.Message.setWrapperField(this, 1, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.clearArtifactType = function() { + return this.setArtifactType(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.hasArtifactType = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional bool can_add_fields = 2; + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.getCanAddFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.setCanAddFields = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.clearCanAddFields = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.hasCanAddFields = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional bool can_omit_fields = 5; + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.getCanOmitFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.setCanOmitFields = function(value) { + return jspb.Message.setField(this, 5, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.clearCanOmitFields = function() { + return jspb.Message.setField(this, 5, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.hasCanOmitFields = function() { + return jspb.Message.getField(this, 5) != null; +}; + + +/** + * optional bool can_delete_fields = 3; + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.getCanDeleteFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.setCanDeleteFields = function(value) { + return jspb.Message.setField(this, 3, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.clearCanDeleteFields = function() { + return jspb.Message.setField(this, 3, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.hasCanDeleteFields = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional bool all_fields_match = 4; + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.getAllFieldsMatch = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, true)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.setAllFieldsMatch = function(value) { + return jspb.Message.setField(this, 4, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.clearAllFieldsMatch = function() { + return jspb.Message.setField(this, 4, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.hasAllFieldsMatch = function() { + return jspb.Message.getField(this, 4) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 6; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 6)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this +*/ +proto.ml_metadata.PutArtifactTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 6, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 6) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutArtifactTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutArtifactTypeResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutArtifactTypeResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactTypeResponse.toObject = function(includeInstance, msg) { + var f, obj = { + typeId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutArtifactTypeResponse} + */ +proto.ml_metadata.PutArtifactTypeResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutArtifactTypeResponse; + return proto.ml_metadata.PutArtifactTypeResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutArtifactTypeResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutArtifactTypeResponse} + */ +proto.ml_metadata.PutArtifactTypeResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt64()); + msg.setTypeId(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutArtifactTypeResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutArtifactTypeResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutArtifactTypeResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutArtifactTypeResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( + 1, + f + ); + } +}; + + +/** + * optional int64 type_id = 1; + * @return {number} + */ +proto.ml_metadata.PutArtifactTypeResponse.prototype.getTypeId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.PutArtifactTypeResponse} returns this + */ +proto.ml_metadata.PutArtifactTypeResponse.prototype.setTypeId = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutArtifactTypeResponse} returns this + */ +proto.ml_metadata.PutArtifactTypeResponse.prototype.clearTypeId = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutArtifactTypeResponse.prototype.hasTypeId = function() { + return jspb.Message.getField(this, 1) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutExecutionsRequest.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionsRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionsRequest.toObject = function(includeInstance, msg) { + var f, obj = { + executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), + ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f), + updateMask: (f = msg.getUpdateMask()) && google_protobuf_field_mask_pb.FieldMask.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionsRequest} + */ +proto.ml_metadata.PutExecutionsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionsRequest; + return proto.ml_metadata.PutExecutionsRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionsRequest} + */ +proto.ml_metadata.PutExecutionsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.addExecutions(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + case 3: + var value = new google_protobuf_field_mask_pb.FieldMask; + reader.readMessage(value,google_protobuf_field_mask_pb.FieldMask.deserializeBinaryFromReader); + msg.setUpdateMask(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExecutionsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } + f = message.getUpdateMask(); + if (f != null) { + writer.writeMessage( + 3, + f, + google_protobuf_field_mask_pb.FieldMask.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated Execution executions = 1; + * @return {!Array} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.getExecutionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutExecutionsRequest} returns this +*/ +proto.ml_metadata.PutExecutionsRequest.prototype.setExecutionsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.Execution=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Execution} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.addExecutions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutExecutionsRequest} returns this + */ +proto.ml_metadata.PutExecutionsRequest.prototype.clearExecutionsList = function() { + return this.setExecutionsList([]); +}; + + +/** + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutExecutionsRequest} returns this +*/ +proto.ml_metadata.PutExecutionsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionsRequest} returns this + */ +proto.ml_metadata.PutExecutionsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional google.protobuf.FieldMask update_mask = 3; + * @return {?proto.google.protobuf.FieldMask} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.getUpdateMask = function() { + return /** @type{?proto.google.protobuf.FieldMask} */ ( + jspb.Message.getWrapperField(this, google_protobuf_field_mask_pb.FieldMask, 3)); +}; + + +/** + * @param {?proto.google.protobuf.FieldMask|undefined} value + * @return {!proto.ml_metadata.PutExecutionsRequest} returns this +*/ +proto.ml_metadata.PutExecutionsRequest.prototype.setUpdateMask = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionsRequest} returns this + */ +proto.ml_metadata.PutExecutionsRequest.prototype.clearUpdateMask = function() { + return this.setUpdateMask(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionsRequest.prototype.hasUpdateMask = function() { + return jspb.Message.getField(this, 3) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutExecutionsResponse.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionsResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionsResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionsResponse.toObject = function(includeInstance, msg) { + var f, obj = { + executionIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionsResponse} + */ +proto.ml_metadata.PutExecutionsResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionsResponse; + return proto.ml_metadata.PutExecutionsResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionsResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionsResponse} + */ +proto.ml_metadata.PutExecutionsResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addExecutionIds(values[i]); + } + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExecutionIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 1, + f + ); + } +}; + + +/** + * repeated int64 execution_ids = 1; + * @return {!Array} + */ +proto.ml_metadata.PutExecutionsResponse.prototype.getExecutionIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutExecutionsResponse} returns this + */ +proto.ml_metadata.PutExecutionsResponse.prototype.setExecutionIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutExecutionsResponse} returns this + */ +proto.ml_metadata.PutExecutionsResponse.prototype.addExecutionIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutExecutionsResponse} returns this + */ +proto.ml_metadata.PutExecutionsResponse.prototype.clearExecutionIdsList = function() { + return this.setExecutionIdsList([]); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionTypeRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionTypeRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionTypeRequest.toObject = function(includeInstance, msg) { + var f, obj = { + executionType: (f = msg.getExecutionType()) && ml_metadata_proto_metadata_store_pb.ExecutionType.toObject(includeInstance, f), + canAddFields: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f, + canOmitFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, + canDeleteFields: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, + allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 4, true), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} + */ +proto.ml_metadata.PutExecutionTypeRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionTypeRequest; + return proto.ml_metadata.PutExecutionTypeRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionTypeRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} + */ +proto.ml_metadata.PutExecutionTypeRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); + msg.setExecutionType(value); + break; + case 2: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanAddFields(value); + break; + case 5: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanOmitFields(value); + break; + case 3: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanDeleteFields(value); + break; + case 4: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setAllFieldsMatch(value); + break; + case 6: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionTypeRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionTypeRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionTypeRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExecutionType(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeBool( + 2, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeBool( + 5, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeBool( + 3, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeBool( + 4, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 6, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + +/** + * optional ExecutionType execution_type = 1; + * @return {?proto.ml_metadata.ExecutionType} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.getExecutionType = function() { + return /** @type{?proto.ml_metadata.ExecutionType} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); +}; + + +/** + * @param {?proto.ml_metadata.ExecutionType|undefined} value + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this +*/ +proto.ml_metadata.PutExecutionTypeRequest.prototype.setExecutionType = function(value) { + return jspb.Message.setWrapperField(this, 1, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.clearExecutionType = function() { + return this.setExecutionType(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.hasExecutionType = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional bool can_add_fields = 2; + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.getCanAddFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.setCanAddFields = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanAddFields = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.hasCanAddFields = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional bool can_omit_fields = 5; + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.getCanOmitFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.setCanOmitFields = function(value) { + return jspb.Message.setField(this, 5, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanOmitFields = function() { + return jspb.Message.setField(this, 5, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.hasCanOmitFields = function() { + return jspb.Message.getField(this, 5) != null; +}; + + +/** + * optional bool can_delete_fields = 3; + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.getCanDeleteFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.setCanDeleteFields = function(value) { + return jspb.Message.setField(this, 3, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanDeleteFields = function() { + return jspb.Message.setField(this, 3, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.hasCanDeleteFields = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional bool all_fields_match = 4; + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.getAllFieldsMatch = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, true)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.setAllFieldsMatch = function(value) { + return jspb.Message.setField(this, 4, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.clearAllFieldsMatch = function() { + return jspb.Message.setField(this, 4, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.hasAllFieldsMatch = function() { + return jspb.Message.getField(this, 4) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 6; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 6)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this +*/ +proto.ml_metadata.PutExecutionTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 6, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 6) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionTypeResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionTypeResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionTypeResponse.toObject = function(includeInstance, msg) { + var f, obj = { + typeId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionTypeResponse} + */ +proto.ml_metadata.PutExecutionTypeResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionTypeResponse; + return proto.ml_metadata.PutExecutionTypeResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionTypeResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionTypeResponse} + */ +proto.ml_metadata.PutExecutionTypeResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt64()); + msg.setTypeId(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionTypeResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionTypeResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionTypeResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionTypeResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( + 1, + f + ); + } +}; + + +/** + * optional int64 type_id = 1; + * @return {number} + */ +proto.ml_metadata.PutExecutionTypeResponse.prototype.getTypeId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.PutExecutionTypeResponse} returns this + */ +proto.ml_metadata.PutExecutionTypeResponse.prototype.setTypeId = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionTypeResponse} returns this + */ +proto.ml_metadata.PutExecutionTypeResponse.prototype.clearTypeId = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionTypeResponse.prototype.hasTypeId = function() { + return jspb.Message.getField(this, 1) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutEventsRequest.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutEventsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutEventsRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutEventsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutEventsRequest.toObject = function(includeInstance, msg) { + var f, obj = { + eventsList: jspb.Message.toObjectList(msg.getEventsList(), + ml_metadata_proto_metadata_store_pb.Event.toObject, includeInstance), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutEventsRequest} + */ +proto.ml_metadata.PutEventsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutEventsRequest; + return proto.ml_metadata.PutEventsRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutEventsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutEventsRequest} + */ +proto.ml_metadata.PutEventsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Event; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); + msg.addEvents(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutEventsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutEventsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutEventsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutEventsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getEventsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated Event events = 1; + * @return {!Array} + */ +proto.ml_metadata.PutEventsRequest.prototype.getEventsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutEventsRequest} returns this +*/ +proto.ml_metadata.PutEventsRequest.prototype.setEventsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.Event=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Event} + */ +proto.ml_metadata.PutEventsRequest.prototype.addEvents = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Event, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutEventsRequest} returns this + */ +proto.ml_metadata.PutEventsRequest.prototype.clearEventsList = function() { + return this.setEventsList([]); +}; + + +/** + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutEventsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutEventsRequest} returns this +*/ +proto.ml_metadata.PutEventsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutEventsRequest} returns this + */ +proto.ml_metadata.PutEventsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutEventsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutEventsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutEventsResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutEventsResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutEventsResponse.toObject = function(includeInstance, msg) { + var f, obj = { + + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutEventsResponse} + */ +proto.ml_metadata.PutEventsResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutEventsResponse; + return proto.ml_metadata.PutEventsResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutEventsResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutEventsResponse} + */ +proto.ml_metadata.PutEventsResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutEventsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutEventsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutEventsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutEventsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutExecutionRequest.repeatedFields_ = [2,3]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionRequest.toObject = function(includeInstance, msg) { + var f, obj = { + execution: (f = msg.getExecution()) && ml_metadata_proto_metadata_store_pb.Execution.toObject(includeInstance, f), + artifactEventPairsList: jspb.Message.toObjectList(msg.getArtifactEventPairsList(), + proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.toObject, includeInstance), + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), + options: (f = msg.getOptions()) && proto.ml_metadata.PutExecutionRequest.Options.toObject(includeInstance, f), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionRequest} + */ +proto.ml_metadata.PutExecutionRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionRequest; + return proto.ml_metadata.PutExecutionRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionRequest} + */ +proto.ml_metadata.PutExecutionRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.setExecution(value); + break; + case 2: + var value = new proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent; + reader.readMessage(value,proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinaryFromReader); + msg.addArtifactEventPairs(value); + break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addContexts(value); + break; + case 4: + var value = new proto.ml_metadata.PutExecutionRequest.Options; + reader.readMessage(value,proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader); + msg.setOptions(value); + break; + case 5: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExecution(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter + ); + } + f = message.getArtifactEventPairsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 2, + f, + proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.serializeBinaryToWriter + ); + } + f = message.getContextsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter + ); + } + f = message.getOptions(); + if (f != null) { + writer.writeMessage( + 4, + f, + proto.ml_metadata.PutExecutionRequest.Options.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 5, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.toObject = function(includeInstance, msg) { + var f, obj = { + artifact: (f = msg.getArtifact()) && ml_metadata_proto_metadata_store_pb.Artifact.toObject(includeInstance, f), + event: (f = msg.getEvent()) && ml_metadata_proto_metadata_store_pb.Event.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent; + return proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.setArtifact(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.Event; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); + msg.setEvent(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifact(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } + f = message.getEvent(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter + ); + } +}; + + +/** + * optional Artifact artifact = 1; + * @return {?proto.ml_metadata.Artifact} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.getArtifact = function() { + return /** @type{?proto.ml_metadata.Artifact} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +}; + + +/** + * @param {?proto.ml_metadata.Artifact|undefined} value + * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this +*/ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.setArtifact = function(value) { + return jspb.Message.setWrapperField(this, 1, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.clearArtifact = function() { + return this.setArtifact(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.hasArtifact = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional Event event = 2; + * @return {?proto.ml_metadata.Event} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.getEvent = function() { + return /** @type{?proto.ml_metadata.Event} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 2)); +}; + + +/** + * @param {?proto.ml_metadata.Event|undefined} value + * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this +*/ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.setEvent = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.clearEvent = function() { + return this.setEvent(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.hasEvent = function() { + return jspb.Message.getField(this, 2) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionRequest.Options.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionRequest.Options} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionRequest.Options.toObject = function(includeInstance, msg) { + var f, obj = { + reuseContextIfAlreadyExist: (f = jspb.Message.getBooleanField(msg, 1)) == null ? undefined : f, + reuseArtifactIfAlreadyExistByExternalId: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionRequest.Options} + */ +proto.ml_metadata.PutExecutionRequest.Options.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionRequest.Options; + return proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionRequest.Options} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionRequest.Options} + */ +proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setReuseContextIfAlreadyExist(value); + break; + case 2: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setReuseArtifactIfAlreadyExistByExternalId(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionRequest.Options.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionRequest.Options} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionRequest.Options.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {boolean} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeBool( + 1, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeBool( + 2, + f + ); + } +}; + + +/** + * optional bool reuse_context_if_already_exist = 1; + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.getReuseContextIfAlreadyExist = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 1, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutExecutionRequest.Options} returns this + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.setReuseContextIfAlreadyExist = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionRequest.Options} returns this + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.clearReuseContextIfAlreadyExist = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.hasReuseContextIfAlreadyExist = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional bool reuse_artifact_if_already_exist_by_external_id = 2; + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.getReuseArtifactIfAlreadyExistByExternalId = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutExecutionRequest.Options} returns this + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.setReuseArtifactIfAlreadyExistByExternalId = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionRequest.Options} returns this + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.clearReuseArtifactIfAlreadyExistByExternalId = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.Options.prototype.hasReuseArtifactIfAlreadyExistByExternalId = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional Execution execution = 1; + * @return {?proto.ml_metadata.Execution} + */ +proto.ml_metadata.PutExecutionRequest.prototype.getExecution = function() { + return /** @type{?proto.ml_metadata.Execution} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +}; + + +/** + * @param {?proto.ml_metadata.Execution|undefined} value + * @return {!proto.ml_metadata.PutExecutionRequest} returns this +*/ +proto.ml_metadata.PutExecutionRequest.prototype.setExecution = function(value) { + return jspb.Message.setWrapperField(this, 1, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionRequest} returns this + */ +proto.ml_metadata.PutExecutionRequest.prototype.clearExecution = function() { + return this.setExecution(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.prototype.hasExecution = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * repeated ArtifactAndEvent artifact_event_pairs = 2; + * @return {!Array} + */ +proto.ml_metadata.PutExecutionRequest.prototype.getArtifactEventPairsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent, 2)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutExecutionRequest} returns this +*/ +proto.ml_metadata.PutExecutionRequest.prototype.setArtifactEventPairsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 2, value); +}; + + +/** + * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} + */ +proto.ml_metadata.PutExecutionRequest.prototype.addArtifactEventPairs = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutExecutionRequest} returns this + */ +proto.ml_metadata.PutExecutionRequest.prototype.clearArtifactEventPairsList = function() { + return this.setArtifactEventPairsList([]); +}; + + +/** + * repeated Context contexts = 3; + * @return {!Array} + */ +proto.ml_metadata.PutExecutionRequest.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 3)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutExecutionRequest} returns this +*/ +proto.ml_metadata.PutExecutionRequest.prototype.setContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 3, value); +}; + + +/** + * @param {!proto.ml_metadata.Context=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Context} + */ +proto.ml_metadata.PutExecutionRequest.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 3, opt_value, proto.ml_metadata.Context, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutExecutionRequest} returns this + */ +proto.ml_metadata.PutExecutionRequest.prototype.clearContextsList = function() { + return this.setContextsList([]); +}; + + +/** + * optional Options options = 4; + * @return {?proto.ml_metadata.PutExecutionRequest.Options} + */ +proto.ml_metadata.PutExecutionRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.PutExecutionRequest.Options} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.PutExecutionRequest.Options, 4)); +}; + + +/** + * @param {?proto.ml_metadata.PutExecutionRequest.Options|undefined} value + * @return {!proto.ml_metadata.PutExecutionRequest} returns this +*/ +proto.ml_metadata.PutExecutionRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionRequest} returns this + */ +proto.ml_metadata.PutExecutionRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 4) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 5; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutExecutionRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 5)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutExecutionRequest} returns this +*/ +proto.ml_metadata.PutExecutionRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 5, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutExecutionRequest} returns this + */ +proto.ml_metadata.PutExecutionRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 5) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutExecutionResponse.repeatedFields_ = [2,3]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutExecutionResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutExecutionResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutExecutionResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionResponse.toObject = function(includeInstance, msg) { + var f, obj = { + executionId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 2)) == null ? undefined : f, + contextIdsList: (f = jspb.Message.getRepeatedField(msg, 3)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutExecutionResponse} + */ +proto.ml_metadata.PutExecutionResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutExecutionResponse; + return proto.ml_metadata.PutExecutionResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutExecutionResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutExecutionResponse} + */ +proto.ml_metadata.PutExecutionResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt64()); + msg.setExecutionId(value); + break; + case 2: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addArtifactIds(values[i]); + } + break; + case 3: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addContextIds(values[i]); + } + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutExecutionResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutExecutionResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutExecutionResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutExecutionResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( + 1, + f + ); + } + f = message.getArtifactIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 2, + f + ); + } + f = message.getContextIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 3, + f + ); + } +}; + + +/** + * optional int64 execution_id = 1; + * @return {number} + */ +proto.ml_metadata.PutExecutionResponse.prototype.getExecutionId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.setExecutionId = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.clearExecutionId = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutExecutionResponse.prototype.hasExecutionId = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * repeated int64 artifact_ids = 2; + * @return {!Array} + */ +proto.ml_metadata.PutExecutionResponse.prototype.getArtifactIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 2)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.setArtifactIdsList = function(value) { + return jspb.Message.setField(this, 2, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.addArtifactIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 2, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.clearArtifactIdsList = function() { + return this.setArtifactIdsList([]); +}; + + +/** + * repeated int64 context_ids = 3; + * @return {!Array} + */ +proto.ml_metadata.PutExecutionResponse.prototype.getContextIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 3)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.setContextIdsList = function(value) { + return jspb.Message.setField(this, 3, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.addContextIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 3, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutExecutionResponse} returns this + */ +proto.ml_metadata.PutExecutionResponse.prototype.clearContextIdsList = function() { + return this.setContextIdsList([]); +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutLineageSubgraphRequest.repeatedFields_ = [1,2,3,4]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutLineageSubgraphRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutLineageSubgraphRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphRequest.toObject = function(includeInstance, msg) { + var f, obj = { + executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), + ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), + eventEdgesList: jspb.Message.toObjectList(msg.getEventEdgesList(), + proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.toObject, includeInstance), + options: (f = msg.getOptions()) && proto.ml_metadata.PutLineageSubgraphRequest.Options.toObject(includeInstance, f), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} + */ +proto.ml_metadata.PutLineageSubgraphRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutLineageSubgraphRequest; + return proto.ml_metadata.PutLineageSubgraphRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutLineageSubgraphRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} + */ +proto.ml_metadata.PutLineageSubgraphRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.addExecutions(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); + break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addContexts(value); + break; + case 4: + var value = new proto.ml_metadata.PutLineageSubgraphRequest.EventEdge; + reader.readMessage(value,proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.deserializeBinaryFromReader); + msg.addEventEdges(value); + break; + case 5: + var value = new proto.ml_metadata.PutLineageSubgraphRequest.Options; + reader.readMessage(value,proto.ml_metadata.PutLineageSubgraphRequest.Options.deserializeBinaryFromReader); + msg.setOptions(value); + break; + case 6: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutLineageSubgraphRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutLineageSubgraphRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExecutionsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter + ); + } + f = message.getArtifactsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } + f = message.getContextsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter + ); + } + f = message.getEventEdgesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 4, + f, + proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.serializeBinaryToWriter + ); + } + f = message.getOptions(); + if (f != null) { + writer.writeMessage( + 5, + f, + proto.ml_metadata.PutLineageSubgraphRequest.Options.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 6, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.toObject = function(includeInstance, msg) { + var f, obj = { + executionIndex: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + artifactIndex: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + event: (f = msg.getEvent()) && ml_metadata_proto_metadata_store_pb.Event.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutLineageSubgraphRequest.EventEdge; + return proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt32()); + msg.setExecutionIndex(value); + break; + case 2: + var value = /** @type {number} */ (reader.readInt32()); + msg.setArtifactIndex(value); + break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.Event; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); + msg.setEvent(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt32( + 1, + f + ); + } + f = /** @type {number} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeInt32( + 2, + f + ); + } + f = message.getEvent(); + if (f != null) { + writer.writeMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter + ); + } +}; + + +/** + * optional int32 execution_index = 1; + * @return {number} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.getExecutionIndex = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.setExecutionIndex = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.clearExecutionIndex = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.hasExecutionIndex = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional int32 artifact_index = 2; + * @return {number} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.getArtifactIndex = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.setArtifactIndex = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.clearArtifactIndex = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.hasArtifactIndex = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional Event event = 3; + * @return {?proto.ml_metadata.Event} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.getEvent = function() { + return /** @type{?proto.ml_metadata.Event} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 3)); +}; + + +/** + * @param {?proto.ml_metadata.Event|undefined} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} returns this +*/ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.setEvent = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.clearEvent = function() { + return this.setEvent(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.EventEdge.prototype.hasEvent = function() { + return jspb.Message.getField(this, 3) != null; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutLineageSubgraphRequest.Options.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutLineageSubgraphRequest.Options} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.toObject = function(includeInstance, msg) { + var f, obj = { + reuseContextIfAlreadyExist: (f = jspb.Message.getBooleanField(msg, 1)) == null ? undefined : f, + reuseArtifactIfAlreadyExistByExternalId: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.Options} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutLineageSubgraphRequest.Options; + return proto.ml_metadata.PutLineageSubgraphRequest.Options.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutLineageSubgraphRequest.Options} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.Options} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setReuseContextIfAlreadyExist(value); + break; + case 2: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setReuseArtifactIfAlreadyExistByExternalId(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutLineageSubgraphRequest.Options.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutLineageSubgraphRequest.Options} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {boolean} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeBool( + 1, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeBool( + 2, + f + ); + } +}; + + +/** + * optional bool reuse_context_if_already_exist = 1; + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.getReuseContextIfAlreadyExist = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 1, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.Options} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.setReuseContextIfAlreadyExist = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.Options} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.clearReuseContextIfAlreadyExist = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.hasReuseContextIfAlreadyExist = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional bool reuse_artifact_if_already_exist_by_external_id = 2; + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.getReuseArtifactIfAlreadyExistByExternalId = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.Options} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.setReuseArtifactIfAlreadyExistByExternalId = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.Options} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.clearReuseArtifactIfAlreadyExistByExternalId = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.Options.prototype.hasReuseArtifactIfAlreadyExistByExternalId = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * repeated Execution executions = 1; + * @return {!Array} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.getExecutionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this +*/ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.setExecutionsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.Execution=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Execution} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.addExecutions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.clearExecutionsList = function() { + return this.setExecutionsList([]); +}; + + +/** + * repeated Artifact artifacts = 2; + * @return {!Array} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 2)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this +*/ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.setArtifactsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 2, value); +}; + + +/** + * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Artifact} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.Artifact, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); +}; + + +/** + * repeated Context contexts = 3; + * @return {!Array} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 3)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this +*/ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.setContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 3, value); +}; + + +/** + * @param {!proto.ml_metadata.Context=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Context} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 3, opt_value, proto.ml_metadata.Context, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.clearContextsList = function() { + return this.setContextsList([]); +}; + + +/** + * repeated EventEdge event_edges = 4; + * @return {!Array} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.getEventEdgesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.ml_metadata.PutLineageSubgraphRequest.EventEdge, 4)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this +*/ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.setEventEdgesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 4, value); +}; + + +/** + * @param {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutLineageSubgraphRequest.EventEdge} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.addEventEdges = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 4, opt_value, proto.ml_metadata.PutLineageSubgraphRequest.EventEdge, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.clearEventEdgesList = function() { + return this.setEventEdgesList([]); +}; + + +/** + * optional Options options = 5; + * @return {?proto.ml_metadata.PutLineageSubgraphRequest.Options} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.PutLineageSubgraphRequest.Options} */ ( + jspb.Message.getWrapperField(this, proto.ml_metadata.PutLineageSubgraphRequest.Options, 5)); +}; + + +/** + * @param {?proto.ml_metadata.PutLineageSubgraphRequest.Options|undefined} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this +*/ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 5, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 5) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 6; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 6)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this +*/ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 6, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutLineageSubgraphRequest} returns this + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutLineageSubgraphRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 6) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutLineageSubgraphResponse.repeatedFields_ = [1,2,3]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutLineageSubgraphResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutLineageSubgraphResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphResponse.toObject = function(includeInstance, msg) { + var f, obj = { + executionIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 2)) == null ? undefined : f, + contextIdsList: (f = jspb.Message.getRepeatedField(msg, 3)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} + */ +proto.ml_metadata.PutLineageSubgraphResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutLineageSubgraphResponse; + return proto.ml_metadata.PutLineageSubgraphResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutLineageSubgraphResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} + */ +proto.ml_metadata.PutLineageSubgraphResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addExecutionIds(values[i]); + } + break; + case 2: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addArtifactIds(values[i]); + } + break; + case 3: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addContextIds(values[i]); + } + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutLineageSubgraphResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutLineageSubgraphResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutLineageSubgraphResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExecutionIdsList(); + if (f.length > 0) { + writer.writePackedInt64( + 1, + f + ); + } + f = message.getArtifactIdsList(); + if (f.length > 0) { + writer.writePackedInt64( + 2, + f + ); + } + f = message.getContextIdsList(); + if (f.length > 0) { + writer.writePackedInt64( + 3, + f + ); + } +}; + + +/** + * repeated int64 execution_ids = 1; + * @return {!Array} + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.getExecutionIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.setExecutionIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.addExecutionIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.clearExecutionIdsList = function() { + return this.setExecutionIdsList([]); +}; + + +/** + * repeated int64 artifact_ids = 2; + * @return {!Array} + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.getArtifactIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 2)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.setArtifactIdsList = function(value) { + return jspb.Message.setField(this, 2, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.addArtifactIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 2, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.clearArtifactIdsList = function() { + return this.setArtifactIdsList([]); +}; + + +/** + * repeated int64 context_ids = 3; + * @return {!Array} + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.getContextIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 3)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.setContextIdsList = function(value) { + return jspb.Message.setField(this, 3, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.addContextIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 3, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutLineageSubgraphResponse} returns this + */ +proto.ml_metadata.PutLineageSubgraphResponse.prototype.clearContextIdsList = function() { + return this.setContextIdsList([]); +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutTypesRequest.repeatedFields_ = [1,2,3]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutTypesRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutTypesRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutTypesRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutTypesRequest.toObject = function(includeInstance, msg) { + var f, obj = { + artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), + ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance), + executionTypesList: jspb.Message.toObjectList(msg.getExecutionTypesList(), + ml_metadata_proto_metadata_store_pb.ExecutionType.toObject, includeInstance), + contextTypesList: jspb.Message.toObjectList(msg.getContextTypesList(), + ml_metadata_proto_metadata_store_pb.ContextType.toObject, includeInstance), + canAddFields: (f = jspb.Message.getBooleanField(msg, 4)) == null ? undefined : f, + canOmitFields: (f = jspb.Message.getBooleanField(msg, 7)) == null ? undefined : f, + canDeleteFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, + allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 6, true), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutTypesRequest} + */ +proto.ml_metadata.PutTypesRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutTypesRequest; + return proto.ml_metadata.PutTypesRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutTypesRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutTypesRequest} + */ +proto.ml_metadata.PutTypesRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.addArtifactTypes(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); + msg.addExecutionTypes(value); + break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.ContextType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); + msg.addContextTypes(value); + break; + case 4: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanAddFields(value); + break; + case 7: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanOmitFields(value); + break; + case 5: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanDeleteFields(value); + break; + case 6: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setAllFieldsMatch(value); + break; + case 8: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutTypesRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutTypesRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutTypesRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutTypesRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactTypesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ); + } + f = message.getExecutionTypesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter + ); + } + f = message.getContextTypesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeBool( + 4, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 7)); + if (f != null) { + writer.writeBool( + 7, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeBool( + 5, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 6)); + if (f != null) { + writer.writeBool( + 6, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 8, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated ArtifactType artifact_types = 1; + * @return {!Array} + */ +proto.ml_metadata.PutTypesRequest.prototype.getArtifactTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this +*/ +proto.ml_metadata.PutTypesRequest.prototype.setArtifactTypesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.ArtifactType=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ArtifactType} + */ +proto.ml_metadata.PutTypesRequest.prototype.addArtifactTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactType, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearArtifactTypesList = function() { + return this.setArtifactTypesList([]); +}; + + +/** + * repeated ExecutionType execution_types = 2; + * @return {!Array} + */ +proto.ml_metadata.PutTypesRequest.prototype.getExecutionTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 2)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this +*/ +proto.ml_metadata.PutTypesRequest.prototype.setExecutionTypesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 2, value); +}; + + +/** + * @param {!proto.ml_metadata.ExecutionType=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ExecutionType} + */ +proto.ml_metadata.PutTypesRequest.prototype.addExecutionTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.ExecutionType, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearExecutionTypesList = function() { + return this.setExecutionTypesList([]); +}; + + +/** + * repeated ContextType context_types = 3; + * @return {!Array} + */ +proto.ml_metadata.PutTypesRequest.prototype.getContextTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 3)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this +*/ +proto.ml_metadata.PutTypesRequest.prototype.setContextTypesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 3, value); +}; + + +/** + * @param {!proto.ml_metadata.ContextType=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ContextType} + */ +proto.ml_metadata.PutTypesRequest.prototype.addContextTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 3, opt_value, proto.ml_metadata.ContextType, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearContextTypesList = function() { + return this.setContextTypesList([]); +}; + + +/** + * optional bool can_add_fields = 4; + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.getCanAddFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.setCanAddFields = function(value) { + return jspb.Message.setField(this, 4, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearCanAddFields = function() { + return jspb.Message.setField(this, 4, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.hasCanAddFields = function() { + return jspb.Message.getField(this, 4) != null; +}; + + +/** + * optional bool can_omit_fields = 7; + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.getCanOmitFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 7, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.setCanOmitFields = function(value) { + return jspb.Message.setField(this, 7, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearCanOmitFields = function() { + return jspb.Message.setField(this, 7, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.hasCanOmitFields = function() { + return jspb.Message.getField(this, 7) != null; +}; + + +/** + * optional bool can_delete_fields = 5; + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.getCanDeleteFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.setCanDeleteFields = function(value) { + return jspb.Message.setField(this, 5, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearCanDeleteFields = function() { + return jspb.Message.setField(this, 5, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.hasCanDeleteFields = function() { + return jspb.Message.getField(this, 5) != null; +}; + + +/** + * optional bool all_fields_match = 6; + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.getAllFieldsMatch = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 6, true)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.setAllFieldsMatch = function(value) { + return jspb.Message.setField(this, 6, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearAllFieldsMatch = function() { + return jspb.Message.setField(this, 6, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.hasAllFieldsMatch = function() { + return jspb.Message.getField(this, 6) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 8; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.PutTypesRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 8)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutTypesRequest} returns this +*/ +proto.ml_metadata.PutTypesRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 8, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutTypesRequest} returns this + */ +proto.ml_metadata.PutTypesRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutTypesRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 8) != null; +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutTypesResponse.repeatedFields_ = [1,2,3]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutTypesResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutTypesResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutTypesResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutTypesResponse.toObject = function(includeInstance, msg) { + var f, obj = { + artifactTypeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + executionTypeIdsList: (f = jspb.Message.getRepeatedField(msg, 2)) == null ? undefined : f, + contextTypeIdsList: (f = jspb.Message.getRepeatedField(msg, 3)) == null ? undefined : f + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutTypesResponse} + */ +proto.ml_metadata.PutTypesResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutTypesResponse; + return proto.ml_metadata.PutTypesResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutTypesResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutTypesResponse} + */ +proto.ml_metadata.PutTypesResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addArtifactTypeIds(values[i]); + } + break; + case 2: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addExecutionTypeIds(values[i]); + } + break; + case 3: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addContextTypeIds(values[i]); + } + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutTypesResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutTypesResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutTypesResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutTypesResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactTypeIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 1, + f + ); + } + f = message.getExecutionTypeIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 2, + f + ); + } + f = message.getContextTypeIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 3, + f + ); + } +}; + + +/** + * repeated int64 artifact_type_ids = 1; + * @return {!Array} + */ +proto.ml_metadata.PutTypesResponse.prototype.getArtifactTypeIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.setArtifactTypeIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.addArtifactTypeIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.clearArtifactTypeIdsList = function() { + return this.setArtifactTypeIdsList([]); +}; + + +/** + * repeated int64 execution_type_ids = 2; + * @return {!Array} + */ +proto.ml_metadata.PutTypesResponse.prototype.getExecutionTypeIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 2)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.setExecutionTypeIdsList = function(value) { + return jspb.Message.setField(this, 2, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.addExecutionTypeIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 2, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.clearExecutionTypeIdsList = function() { + return this.setExecutionTypeIdsList([]); +}; + + +/** + * repeated int64 context_type_ids = 3; + * @return {!Array} + */ +proto.ml_metadata.PutTypesResponse.prototype.getContextTypeIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 3)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.setContextTypeIdsList = function(value) { + return jspb.Message.setField(this, 3, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.addContextTypeIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 3, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutTypesResponse} returns this + */ +proto.ml_metadata.PutTypesResponse.prototype.clearContextTypeIdsList = function() { + return this.setContextTypeIdsList([]); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.PutContextTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutContextTypeRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutContextTypeRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.PutContextTypeRequest.toObject = function(includeInstance, msg) { + var f, obj = { + contextType: (f = msg.getContextType()) && ml_metadata_proto_metadata_store_pb.ContextType.toObject(includeInstance, f), + canAddFields: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f, + canOmitFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, + canDeleteFields: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, + allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 4, true), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutContextTypeRequest} + */ +proto.ml_metadata.PutContextTypeRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutContextTypeRequest; + return proto.ml_metadata.PutContextTypeRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutContextTypeRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutContextTypeRequest} + */ +proto.ml_metadata.PutContextTypeRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ContextType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); + msg.setContextType(value); + break; + case 2: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanAddFields(value); + break; + case 5: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanOmitFields(value); + break; + case 3: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setCanDeleteFields(value); + break; + case 4: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setAllFieldsMatch(value); + break; + case 6: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.PutContextTypeRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutContextTypeRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; - + /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ArtifactAndType} message + * @param {!proto.ml_metadata.PutContextTypeRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ArtifactAndType.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PutContextTypeRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifact(); + f = message.getContextType(); if (f != null) { writer.writeMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter ); } - f = message.getType(); + f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeMessage( + writer.writeBool( 2, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeBool( + 5, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeBool( + 3, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); + if (f != null) { + writer.writeBool( + 4, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 6, f, - ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional Artifact artifact = 1; - * @return {?proto.ml_metadata.Artifact} + * optional ContextType context_type = 1; + * @return {?proto.ml_metadata.ContextType} */ -proto.ml_metadata.ArtifactAndType.prototype.getArtifact = function() { - return /** @type{?proto.ml_metadata.Artifact} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.PutContextTypeRequest.prototype.getContextType = function() { + return /** @type{?proto.ml_metadata.ContextType} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); }; /** - * @param {?proto.ml_metadata.Artifact|undefined} value - * @return {!proto.ml_metadata.ArtifactAndType} returns this + * @param {?proto.ml_metadata.ContextType|undefined} value + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this */ -proto.ml_metadata.ArtifactAndType.prototype.setArtifact = function(value) { +proto.ml_metadata.PutContextTypeRequest.prototype.setContextType = function(value) { return jspb.Message.setWrapperField(this, 1, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ArtifactAndType} returns this + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this */ -proto.ml_metadata.ArtifactAndType.prototype.clearArtifact = function() { - return this.setArtifact(undefined); +proto.ml_metadata.PutContextTypeRequest.prototype.clearContextType = function() { + return this.setContextType(undefined); }; @@ -2180,36 +8882,107 @@ proto.ml_metadata.ArtifactAndType.prototype.clearArtifact = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ArtifactAndType.prototype.hasArtifact = function() { +proto.ml_metadata.PutContextTypeRequest.prototype.hasContextType = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional ArtifactType type = 2; - * @return {?proto.ml_metadata.ArtifactType} + * optional bool can_add_fields = 2; + * @return {boolean} + */ +proto.ml_metadata.PutContextTypeRequest.prototype.getCanAddFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + */ +proto.ml_metadata.PutContextTypeRequest.prototype.setCanAddFields = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + */ +proto.ml_metadata.PutContextTypeRequest.prototype.clearCanAddFields = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutContextTypeRequest.prototype.hasCanAddFields = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional bool can_omit_fields = 5; + * @return {boolean} + */ +proto.ml_metadata.PutContextTypeRequest.prototype.getCanOmitFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + */ +proto.ml_metadata.PutContextTypeRequest.prototype.setCanOmitFields = function(value) { + return jspb.Message.setField(this, 5, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + */ +proto.ml_metadata.PutContextTypeRequest.prototype.clearCanOmitFields = function() { + return jspb.Message.setField(this, 5, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.PutContextTypeRequest.prototype.hasCanOmitFields = function() { + return jspb.Message.getField(this, 5) != null; +}; + + +/** + * optional bool can_delete_fields = 3; + * @return {boolean} */ -proto.ml_metadata.ArtifactAndType.prototype.getType = function() { - return /** @type{?proto.ml_metadata.ArtifactType} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 2)); +proto.ml_metadata.PutContextTypeRequest.prototype.getCanDeleteFields = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); }; /** - * @param {?proto.ml_metadata.ArtifactType|undefined} value - * @return {!proto.ml_metadata.ArtifactAndType} returns this -*/ -proto.ml_metadata.ArtifactAndType.prototype.setType = function(value) { - return jspb.Message.setWrapperField(this, 2, value); + * @param {boolean} value + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + */ +proto.ml_metadata.PutContextTypeRequest.prototype.setCanDeleteFields = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ArtifactAndType} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this */ -proto.ml_metadata.ArtifactAndType.prototype.clearType = function() { - return this.setType(undefined); +proto.ml_metadata.PutContextTypeRequest.prototype.clearCanDeleteFields = function() { + return jspb.Message.setField(this, 3, undefined); }; @@ -2217,151 +8990,84 @@ proto.ml_metadata.ArtifactAndType.prototype.clearType = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ArtifactAndType.prototype.hasType = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.PutContextTypeRequest.prototype.hasCanDeleteFields = function() { + return jspb.Message.getField(this, 3) != null; }; - - - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * optional bool all_fields_match = 4; + * @return {boolean} */ -proto.ml_metadata.ArtifactStructMap.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ArtifactStructMap.toObject(opt_includeInstance, this); +proto.ml_metadata.PutContextTypeRequest.prototype.getAllFieldsMatch = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, true)); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ArtifactStructMap} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * @param {boolean} value + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this */ -proto.ml_metadata.ArtifactStructMap.toObject = function(includeInstance, msg) { - var f, obj = { - propertiesMap: (f = msg.getPropertiesMap()) ? f.toObject(includeInstance, proto.ml_metadata.ArtifactStruct.toObject) : [] - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +proto.ml_metadata.PutContextTypeRequest.prototype.setAllFieldsMatch = function(value) { + return jspb.Message.setField(this, 4, value); }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ArtifactStructMap} + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this */ -proto.ml_metadata.ArtifactStructMap.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ArtifactStructMap; - return proto.ml_metadata.ArtifactStructMap.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.PutContextTypeRequest.prototype.clearAllFieldsMatch = function() { + return jspb.Message.setField(this, 4, undefined); }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.ArtifactStructMap} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ArtifactStructMap} + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.ArtifactStructMap.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = msg.getPropertiesMap(); - reader.readMessage(value, function(message, reader) { - jspb.Map.deserializeBinary(message, reader, jspb.BinaryReader.prototype.readString, jspb.BinaryReader.prototype.readMessage, proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader, "", new proto.ml_metadata.ArtifactStruct()); - }); - break; - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.PutContextTypeRequest.prototype.hasAllFieldsMatch = function() { + return jspb.Message.getField(this, 4) != null; }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * optional TransactionOptions transaction_options = 6; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.ArtifactStructMap.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ArtifactStructMap.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.PutContextTypeRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 6)); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ArtifactStructMap} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages - */ -proto.ml_metadata.ArtifactStructMap.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getPropertiesMap(true); - if (f && f.getLength() > 0) { - f.serializeBinary(1, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeMessage, proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter); - } + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this +*/ +proto.ml_metadata.PutContextTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 6, value); }; /** - * map properties = 1; - * @param {boolean=} opt_noLazyCreate Do not create the map if - * empty, instead returning `undefined` - * @return {!jspb.Map} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutContextTypeRequest} returns this */ -proto.ml_metadata.ArtifactStructMap.prototype.getPropertiesMap = function(opt_noLazyCreate) { - return /** @type {!jspb.Map} */ ( - jspb.Message.getMapField(this, 1, opt_noLazyCreate, - proto.ml_metadata.ArtifactStruct)); +proto.ml_metadata.PutContextTypeRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; /** - * Clears values from the map. The map will be non-null. - * @return {!proto.ml_metadata.ArtifactStructMap} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.ArtifactStructMap.prototype.clearPropertiesMap = function() { - this.getPropertiesMap().clear(); - return this;}; - +proto.ml_metadata.PutContextTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 6) != null; +}; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.ArtifactStructList.repeatedFields_ = [1]; @@ -2378,8 +9084,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.ArtifactStructList.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ArtifactStructList.toObject(opt_includeInstance, this); +proto.ml_metadata.PutContextTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutContextTypeResponse.toObject(opt_includeInstance, this); }; @@ -2388,14 +9094,13 @@ proto.ml_metadata.ArtifactStructList.prototype.toObject = function(opt_includeIn * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ArtifactStructList} msg The msg instance to transform. + * @param {!proto.ml_metadata.PutContextTypeResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ArtifactStructList.toObject = function(includeInstance, msg) { +proto.ml_metadata.PutContextTypeResponse.toObject = function(includeInstance, msg) { var f, obj = { - elementsList: jspb.Message.toObjectList(msg.getElementsList(), - proto.ml_metadata.ArtifactStruct.toObject, includeInstance) + typeId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f }; if (includeInstance) { @@ -2409,23 +9114,23 @@ proto.ml_metadata.ArtifactStructList.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ArtifactStructList} + * @return {!proto.ml_metadata.PutContextTypeResponse} */ -proto.ml_metadata.ArtifactStructList.deserializeBinary = function(bytes) { +proto.ml_metadata.PutContextTypeResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ArtifactStructList; - return proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PutContextTypeResponse; + return proto.ml_metadata.PutContextTypeResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.ArtifactStructList} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PutContextTypeResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ArtifactStructList} + * @return {!proto.ml_metadata.PutContextTypeResponse} */ -proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PutContextTypeResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -2433,9 +9138,8 @@ proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader = function(msg, var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new proto.ml_metadata.ArtifactStruct; - reader.readMessage(value,proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader); - msg.addElements(value); + var value = /** @type {number} */ (reader.readInt64()); + msg.setTypeId(value); break; default: reader.skipField(); @@ -2450,9 +9154,9 @@ proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader = function(msg, * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.ArtifactStructList.prototype.serializeBinary = function() { +proto.ml_metadata.PutContextTypeResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ArtifactStructList.serializeBinaryToWriter(this, writer); + proto.ml_metadata.PutContextTypeResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -2460,88 +9164,65 @@ proto.ml_metadata.ArtifactStructList.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ArtifactStructList} message + * @param {!proto.ml_metadata.PutContextTypeResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ArtifactStructList.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PutContextTypeResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getElementsList(); - if (f.length > 0) { - writer.writeRepeatedMessage( + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( 1, - f, - proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter + f ); } }; /** - * repeated ArtifactStruct elements = 1; - * @return {!Array} - */ -proto.ml_metadata.ArtifactStructList.prototype.getElementsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, proto.ml_metadata.ArtifactStruct, 1)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.ArtifactStructList} returns this -*/ -proto.ml_metadata.ArtifactStructList.prototype.setElementsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); -}; - - -/** - * @param {!proto.ml_metadata.ArtifactStruct=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.ArtifactStruct} + * optional int64 type_id = 1; + * @return {number} */ -proto.ml_metadata.ArtifactStructList.prototype.addElements = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactStruct, opt_index); +proto.ml_metadata.PutContextTypeResponse.prototype.getTypeId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.ArtifactStructList} returns this + * @param {number} value + * @return {!proto.ml_metadata.PutContextTypeResponse} returns this */ -proto.ml_metadata.ArtifactStructList.prototype.clearElementsList = function() { - return this.setElementsList([]); +proto.ml_metadata.PutContextTypeResponse.prototype.setTypeId = function(value) { + return jspb.Message.setField(this, 1, value); }; - /** - * Oneof group definitions for this message. Each group defines the field - * numbers belonging to that group. When of these fields' value is set, all - * other fields in the group are cleared. During deserialization, if multiple - * fields are encountered for a group, only the last value seen will be kept. - * @private {!Array>} - * @const + * Clears the field making it undefined. + * @return {!proto.ml_metadata.PutContextTypeResponse} returns this */ -proto.ml_metadata.ArtifactStruct.oneofGroups_ = [[1,2,3]]; +proto.ml_metadata.PutContextTypeResponse.prototype.clearTypeId = function() { + return jspb.Message.setField(this, 1, undefined); +}; + /** - * @enum {number} + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.ArtifactStruct.ValueCase = { - VALUE_NOT_SET: 0, - ARTIFACT: 1, - MAP: 2, - LIST: 3 +proto.ml_metadata.PutContextTypeResponse.prototype.hasTypeId = function() { + return jspb.Message.getField(this, 1) != null; }; + + /** - * @return {proto.ml_metadata.ArtifactStruct.ValueCase} + * List of repeated fields within this message type. + * @private {!Array} + * @const */ -proto.ml_metadata.ArtifactStruct.prototype.getValueCase = function() { - return /** @type {proto.ml_metadata.ArtifactStruct.ValueCase} */(jspb.Message.computeOneofCase(this, proto.ml_metadata.ArtifactStruct.oneofGroups_[0])); -}; +proto.ml_metadata.PutContextsRequest.repeatedFields_ = [1]; @@ -2558,8 +9239,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.ArtifactStruct.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.ArtifactStruct.toObject(opt_includeInstance, this); +proto.ml_metadata.PutContextsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutContextsRequest.toObject(opt_includeInstance, this); }; @@ -2568,15 +9249,16 @@ proto.ml_metadata.ArtifactStruct.prototype.toObject = function(opt_includeInstan * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.ArtifactStruct} msg The msg instance to transform. + * @param {!proto.ml_metadata.PutContextsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ArtifactStruct.toObject = function(includeInstance, msg) { +proto.ml_metadata.PutContextsRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifact: (f = msg.getArtifact()) && proto.ml_metadata.ArtifactAndType.toObject(includeInstance, f), - map: (f = msg.getMap()) && proto.ml_metadata.ArtifactStructMap.toObject(includeInstance, f), - list: (f = msg.getList()) && proto.ml_metadata.ArtifactStructList.toObject(includeInstance, f) + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f), + updateMask: (f = msg.getUpdateMask()) && google_protobuf_field_mask_pb.FieldMask.toObject(includeInstance, f) }; if (includeInstance) { @@ -2590,23 +9272,23 @@ proto.ml_metadata.ArtifactStruct.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.ArtifactStruct} + * @return {!proto.ml_metadata.PutContextsRequest} */ -proto.ml_metadata.ArtifactStruct.deserializeBinary = function(bytes) { +proto.ml_metadata.PutContextsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.ArtifactStruct; - return proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PutContextsRequest; + return proto.ml_metadata.PutContextsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.ArtifactStruct} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PutContextsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.ArtifactStruct} + * @return {!proto.ml_metadata.PutContextsRequest} */ -proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PutContextsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -2614,19 +9296,19 @@ proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader = function(msg, rea var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new proto.ml_metadata.ArtifactAndType; - reader.readMessage(value,proto.ml_metadata.ArtifactAndType.deserializeBinaryFromReader); - msg.setArtifact(value); + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addContexts(value); break; case 2: - var value = new proto.ml_metadata.ArtifactStructMap; - reader.readMessage(value,proto.ml_metadata.ArtifactStructMap.deserializeBinaryFromReader); - msg.setMap(value); + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; case 3: - var value = new proto.ml_metadata.ArtifactStructList; - reader.readMessage(value,proto.ml_metadata.ArtifactStructList.deserializeBinaryFromReader); - msg.setList(value); + var value = new google_protobuf_field_mask_pb.FieldMask; + reader.readMessage(value,google_protobuf_field_mask_pb.FieldMask.deserializeBinaryFromReader); + msg.setUpdateMask(value); break; default: reader.skipField(); @@ -2641,9 +9323,9 @@ proto.ml_metadata.ArtifactStruct.deserializeBinaryFromReader = function(msg, rea * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.ArtifactStruct.prototype.serializeBinary = function() { +proto.ml_metadata.PutContextsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter(this, writer); + proto.ml_metadata.PutContextsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -2651,101 +9333,102 @@ proto.ml_metadata.ArtifactStruct.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.ArtifactStruct} message + * @param {!proto.ml_metadata.PutContextsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.ArtifactStruct.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PutContextsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifact(); - if (f != null) { - writer.writeMessage( + f = message.getContextsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, f, - proto.ml_metadata.ArtifactAndType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter ); } - f = message.getMap(); + f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( 2, f, - proto.ml_metadata.ArtifactStructMap.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } - f = message.getList(); + f = message.getUpdateMask(); if (f != null) { writer.writeMessage( 3, f, - proto.ml_metadata.ArtifactStructList.serializeBinaryToWriter + google_protobuf_field_mask_pb.FieldMask.serializeBinaryToWriter ); } }; /** - * optional ArtifactAndType artifact = 1; - * @return {?proto.ml_metadata.ArtifactAndType} + * repeated Context contexts = 1; + * @return {!Array} */ -proto.ml_metadata.ArtifactStruct.prototype.getArtifact = function() { - return /** @type{?proto.ml_metadata.ArtifactAndType} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.ArtifactAndType, 1)); +proto.ml_metadata.PutContextsRequest.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * @param {?proto.ml_metadata.ArtifactAndType|undefined} value - * @return {!proto.ml_metadata.ArtifactStruct} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.PutContextsRequest} returns this */ -proto.ml_metadata.ArtifactStruct.prototype.setArtifact = function(value) { - return jspb.Message.setOneofWrapperField(this, 1, proto.ml_metadata.ArtifactStruct.oneofGroups_[0], value); +proto.ml_metadata.PutContextsRequest.prototype.setContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ArtifactStruct} returns this + * @param {!proto.ml_metadata.Context=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.ArtifactStruct.prototype.clearArtifact = function() { - return this.setArtifact(undefined); +proto.ml_metadata.PutContextsRequest.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutContextsRequest} returns this */ -proto.ml_metadata.ArtifactStruct.prototype.hasArtifact = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.PutContextsRequest.prototype.clearContextsList = function() { + return this.setContextsList([]); }; /** - * optional ArtifactStructMap map = 2; - * @return {?proto.ml_metadata.ArtifactStructMap} + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.ArtifactStruct.prototype.getMap = function() { - return /** @type{?proto.ml_metadata.ArtifactStructMap} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.ArtifactStructMap, 2)); +proto.ml_metadata.PutContextsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** - * @param {?proto.ml_metadata.ArtifactStructMap|undefined} value - * @return {!proto.ml_metadata.ArtifactStruct} returns this + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutContextsRequest} returns this */ -proto.ml_metadata.ArtifactStruct.prototype.setMap = function(value) { - return jspb.Message.setOneofWrapperField(this, 2, proto.ml_metadata.ArtifactStruct.oneofGroups_[0], value); +proto.ml_metadata.PutContextsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ArtifactStruct} returns this + * @return {!proto.ml_metadata.PutContextsRequest} returns this */ -proto.ml_metadata.ArtifactStruct.prototype.clearMap = function() { - return this.setMap(undefined); +proto.ml_metadata.PutContextsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -2753,36 +9436,36 @@ proto.ml_metadata.ArtifactStruct.prototype.clearMap = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ArtifactStruct.prototype.hasMap = function() { +proto.ml_metadata.PutContextsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; /** - * optional ArtifactStructList list = 3; - * @return {?proto.ml_metadata.ArtifactStructList} + * optional google.protobuf.FieldMask update_mask = 3; + * @return {?proto.google.protobuf.FieldMask} */ -proto.ml_metadata.ArtifactStruct.prototype.getList = function() { - return /** @type{?proto.ml_metadata.ArtifactStructList} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.ArtifactStructList, 3)); +proto.ml_metadata.PutContextsRequest.prototype.getUpdateMask = function() { + return /** @type{?proto.google.protobuf.FieldMask} */ ( + jspb.Message.getWrapperField(this, google_protobuf_field_mask_pb.FieldMask, 3)); }; /** - * @param {?proto.ml_metadata.ArtifactStructList|undefined} value - * @return {!proto.ml_metadata.ArtifactStruct} returns this + * @param {?proto.google.protobuf.FieldMask|undefined} value + * @return {!proto.ml_metadata.PutContextsRequest} returns this */ -proto.ml_metadata.ArtifactStruct.prototype.setList = function(value) { - return jspb.Message.setOneofWrapperField(this, 3, proto.ml_metadata.ArtifactStruct.oneofGroups_[0], value); +proto.ml_metadata.PutContextsRequest.prototype.setUpdateMask = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.ArtifactStruct} returns this + * @return {!proto.ml_metadata.PutContextsRequest} returns this */ -proto.ml_metadata.ArtifactStruct.prototype.clearList = function() { - return this.setList(undefined); +proto.ml_metadata.PutContextsRequest.prototype.clearUpdateMask = function() { + return this.setUpdateMask(undefined); }; @@ -2790,7 +9473,7 @@ proto.ml_metadata.ArtifactStruct.prototype.clearList = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.ArtifactStruct.prototype.hasList = function() { +proto.ml_metadata.PutContextsRequest.prototype.hasUpdateMask = function() { return jspb.Message.getField(this, 3) != null; }; @@ -2801,7 +9484,7 @@ proto.ml_metadata.ArtifactStruct.prototype.hasList = function() { * @private {!Array} * @const */ -proto.ml_metadata.PutArtifactsRequest.repeatedFields_ = [1]; +proto.ml_metadata.PutContextsResponse.repeatedFields_ = [1]; @@ -2818,8 +9501,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutArtifactsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutArtifactsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.PutContextsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutContextsResponse.toObject(opt_includeInstance, this); }; @@ -2828,15 +9511,13 @@ proto.ml_metadata.PutArtifactsRequest.prototype.toObject = function(opt_includeI * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutArtifactsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.PutContextsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.PutContextsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), - ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), - options: (f = msg.getOptions()) && proto.ml_metadata.PutArtifactsRequest.Options.toObject(includeInstance, f) + contextIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f }; if (includeInstance) { @@ -2850,23 +9531,23 @@ proto.ml_metadata.PutArtifactsRequest.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutArtifactsRequest} + * @return {!proto.ml_metadata.PutContextsResponse} */ -proto.ml_metadata.PutArtifactsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.PutContextsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutArtifactsRequest; - return proto.ml_metadata.PutArtifactsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PutContextsResponse; + return proto.ml_metadata.PutContextsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutArtifactsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PutContextsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutArtifactsRequest} + * @return {!proto.ml_metadata.PutContextsResponse} */ -proto.ml_metadata.PutArtifactsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PutContextsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -2874,14 +9555,10 @@ proto.ml_metadata.PutArtifactsRequest.deserializeBinaryFromReader = function(msg var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.addArtifacts(value); - break; - case 2: - var value = new proto.ml_metadata.PutArtifactsRequest.Options; - reader.readMessage(value,proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader); - msg.setOptions(value); + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addContextIds(values[i]); + } break; default: reader.skipField(); @@ -2896,9 +9573,9 @@ proto.ml_metadata.PutArtifactsRequest.deserializeBinaryFromReader = function(msg * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutArtifactsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.PutContextsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutArtifactsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.PutContextsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -2906,31 +9583,66 @@ proto.ml_metadata.PutArtifactsRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutArtifactsRequest} message + * @param {!proto.ml_metadata.PutContextsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PutContextsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactsList(); + f = message.getContextIdsList(); if (f.length > 0) { - writer.writeRepeatedMessage( + writer.writeRepeatedInt64( 1, - f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter - ); - } - f = message.getOptions(); - if (f != null) { - writer.writeMessage( - 2, - f, - proto.ml_metadata.PutArtifactsRequest.Options.serializeBinaryToWriter + f ); } }; +/** + * repeated int64 context_ids = 1; + * @return {!Array} + */ +proto.ml_metadata.PutContextsResponse.prototype.getContextIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.PutContextsResponse} returns this + */ +proto.ml_metadata.PutContextsResponse.prototype.setContextIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.PutContextsResponse} returns this + */ +proto.ml_metadata.PutContextsResponse.prototype.addContextIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutContextsResponse} returns this + */ +proto.ml_metadata.PutContextsResponse.prototype.clearContextIdsList = function() { + return this.setContextIdsList([]); +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.PutAttributionsAndAssociationsRequest.repeatedFields_ = [1,2]; @@ -2947,8 +9659,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutArtifactsRequest.Options.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutArtifactsRequest.Options.toObject(opt_includeInstance, this); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutAttributionsAndAssociationsRequest.toObject(opt_includeInstance, this); }; @@ -2957,13 +9669,17 @@ proto.ml_metadata.PutArtifactsRequest.Options.prototype.toObject = function(opt_ * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutArtifactsRequest.Options} msg The msg instance to transform. + * @param {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactsRequest.Options.toObject = function(includeInstance, msg) { +proto.ml_metadata.PutAttributionsAndAssociationsRequest.toObject = function(includeInstance, msg) { var f, obj = { - abortIfLatestUpdatedTimeChanged: (f = jspb.Message.getBooleanField(msg, 1)) == null ? undefined : f + attributionsList: jspb.Message.toObjectList(msg.getAttributionsList(), + ml_metadata_proto_metadata_store_pb.Attribution.toObject, includeInstance), + associationsList: jspb.Message.toObjectList(msg.getAssociationsList(), + ml_metadata_proto_metadata_store_pb.Association.toObject, includeInstance), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -2977,23 +9693,23 @@ proto.ml_metadata.PutArtifactsRequest.Options.toObject = function(includeInstanc /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutArtifactsRequest.Options} + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} */ -proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinary = function(bytes) { +proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutArtifactsRequest.Options; - return proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PutAttributionsAndAssociationsRequest; + return proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutArtifactsRequest.Options} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutArtifactsRequest.Options} + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} */ -proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -3001,8 +9717,19 @@ proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader = func var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setAbortIfLatestUpdatedTimeChanged(value); + var value = new ml_metadata_proto_metadata_store_pb.Attribution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Attribution.deserializeBinaryFromReader); + msg.addAttributions(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.Association; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Association.deserializeBinaryFromReader); + msg.addAssociations(value); + break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -3017,9 +9744,9 @@ proto.ml_metadata.PutArtifactsRequest.Options.deserializeBinaryFromReader = func * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutArtifactsRequest.Options.prototype.serializeBinary = function() { +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutArtifactsRequest.Options.serializeBinaryToWriter(this, writer); + proto.ml_metadata.PutAttributionsAndAssociationsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -3027,121 +9754,140 @@ proto.ml_metadata.PutArtifactsRequest.Options.prototype.serializeBinary = functi /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutArtifactsRequest.Options} message + * @param {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactsRequest.Options.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PutAttributionsAndAssociationsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {boolean} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeBool( + f = message.getAttributionsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, - f + f, + ml_metadata_proto_metadata_store_pb.Attribution.serializeBinaryToWriter + ); + } + f = message.getAssociationsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.Association.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional bool abort_if_latest_updated_time_changed = 1; - * @return {boolean} + * repeated Attribution attributions = 1; + * @return {!Array} */ -proto.ml_metadata.PutArtifactsRequest.Options.prototype.getAbortIfLatestUpdatedTimeChanged = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 1, false)); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.getAttributionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Attribution, 1)); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutArtifactsRequest.Options} returns this - */ -proto.ml_metadata.PutArtifactsRequest.Options.prototype.setAbortIfLatestUpdatedTimeChanged = function(value) { - return jspb.Message.setField(this, 1, value); + * @param {!Array} value + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this +*/ +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.setAttributionsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutArtifactsRequest.Options} returns this + * @param {!proto.ml_metadata.Attribution=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Attribution} */ -proto.ml_metadata.PutArtifactsRequest.Options.prototype.clearAbortIfLatestUpdatedTimeChanged = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.addAttributions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Attribution, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this */ -proto.ml_metadata.PutArtifactsRequest.Options.prototype.hasAbortIfLatestUpdatedTimeChanged = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.clearAttributionsList = function() { + return this.setAttributionsList([]); }; /** - * repeated Artifact artifacts = 1; - * @return {!Array} + * repeated Association associations = 2; + * @return {!Array} */ -proto.ml_metadata.PutArtifactsRequest.prototype.getArtifactsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.getAssociationsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Association, 2)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this */ -proto.ml_metadata.PutArtifactsRequest.prototype.setArtifactsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.setAssociationsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 2, value); }; /** - * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {!proto.ml_metadata.Association=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Artifact} + * @return {!proto.ml_metadata.Association} */ -proto.ml_metadata.PutArtifactsRequest.prototype.addArtifacts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.addAssociations = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.Association, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this */ -proto.ml_metadata.PutArtifactsRequest.prototype.clearArtifactsList = function() { - return this.setArtifactsList([]); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.clearAssociationsList = function() { + return this.setAssociationsList([]); }; /** - * optional Options options = 2; - * @return {?proto.ml_metadata.PutArtifactsRequest.Options} + * optional TransactionOptions transaction_options = 3; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutArtifactsRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.PutArtifactsRequest.Options} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.PutArtifactsRequest.Options, 2)); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); }; /** - * @param {?proto.ml_metadata.PutArtifactsRequest.Options|undefined} value - * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this */ -proto.ml_metadata.PutArtifactsRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutArtifactsRequest} returns this + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this */ -proto.ml_metadata.PutArtifactsRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -3149,19 +9895,12 @@ proto.ml_metadata.PutArtifactsRequest.prototype.clearOptions = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutArtifactsRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.PutArtifactsResponse.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -3177,8 +9916,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutArtifactsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutArtifactsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.PutAttributionsAndAssociationsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutAttributionsAndAssociationsResponse.toObject(opt_includeInstance, this); }; @@ -3187,13 +9926,13 @@ proto.ml_metadata.PutArtifactsResponse.prototype.toObject = function(opt_include * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutArtifactsResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactsResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.PutAttributionsAndAssociationsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f + }; if (includeInstance) { @@ -3207,111 +9946,68 @@ proto.ml_metadata.PutArtifactsResponse.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutArtifactsResponse} + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} */ -proto.ml_metadata.PutArtifactsResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.PutAttributionsAndAssociationsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutArtifactsResponse; - return proto.ml_metadata.PutArtifactsResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PutAttributionsAndAssociationsResponse; + return proto.ml_metadata.PutAttributionsAndAssociationsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutArtifactsResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutArtifactsResponse} + * @return {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} */ -proto.ml_metadata.PutArtifactsResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PutAttributionsAndAssociationsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { - case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addArtifactIds(values[i]); - } - break; default: reader.skipField(); break; } } - return msg; -}; - - -/** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} - */ -proto.ml_metadata.PutArtifactsResponse.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutArtifactsResponse.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); -}; - - -/** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutArtifactsResponse} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages - */ -proto.ml_metadata.PutArtifactsResponse.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getArtifactIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( - 1, - f - ); - } -}; - - -/** - * repeated int64 artifact_ids = 1; - * @return {!Array} - */ -proto.ml_metadata.PutArtifactsResponse.prototype.getArtifactIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); + return msg; }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutArtifactsResponse} returns this + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.PutArtifactsResponse.prototype.setArtifactIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); +proto.ml_metadata.PutAttributionsAndAssociationsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutAttributionsAndAssociationsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.PutArtifactsResponse} returns this + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactsResponse.prototype.addArtifactIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.PutAttributionsAndAssociationsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; }; + /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutArtifactsResponse} returns this + * List of repeated fields within this message type. + * @private {!Array} + * @const */ -proto.ml_metadata.PutArtifactsResponse.prototype.clearArtifactIdsList = function() { - return this.setArtifactIdsList([]); -}; - - +proto.ml_metadata.PutParentContextsRequest.repeatedFields_ = [1]; @@ -3328,8 +10024,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutArtifactTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.PutParentContextsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutParentContextsRequest.toObject(opt_includeInstance, this); }; @@ -3338,17 +10034,15 @@ proto.ml_metadata.PutArtifactTypeRequest.prototype.toObject = function(opt_inclu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutArtifactTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.PutParentContextsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.PutParentContextsRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifactType: (f = msg.getArtifactType()) && ml_metadata_proto_metadata_store_pb.ArtifactType.toObject(includeInstance, f), - canAddFields: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f, - canOmitFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, - canDeleteFields: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, - allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 4, true) + parentContextsList: jspb.Message.toObjectList(msg.getParentContextsList(), + ml_metadata_proto_metadata_store_pb.ParentContext.toObject, includeInstance), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -3362,23 +10056,23 @@ proto.ml_metadata.PutArtifactTypeRequest.toObject = function(includeInstance, ms /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutArtifactTypeRequest} + * @return {!proto.ml_metadata.PutParentContextsRequest} */ -proto.ml_metadata.PutArtifactTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.PutParentContextsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutArtifactTypeRequest; - return proto.ml_metadata.PutArtifactTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.PutParentContextsRequest; + return proto.ml_metadata.PutParentContextsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutArtifactTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.PutParentContextsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutArtifactTypeRequest} + * @return {!proto.ml_metadata.PutParentContextsRequest} */ -proto.ml_metadata.PutArtifactTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.PutParentContextsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -3386,25 +10080,14 @@ proto.ml_metadata.PutArtifactTypeRequest.deserializeBinaryFromReader = function( var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); - msg.setArtifactType(value); + var value = new ml_metadata_proto_metadata_store_pb.ParentContext; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ParentContext.deserializeBinaryFromReader); + msg.addParentContexts(value); break; case 2: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanAddFields(value); - break; - case 5: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanOmitFields(value); - break; - case 3: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanDeleteFields(value); - break; - case 4: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setAllFieldsMatch(value); + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -3419,9 +10102,9 @@ proto.ml_metadata.PutArtifactTypeRequest.deserializeBinaryFromReader = function( * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.PutParentContextsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutArtifactTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.PutParentContextsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -3429,112 +10112,94 @@ proto.ml_metadata.PutArtifactTypeRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutArtifactTypeRequest} message + * @param {!proto.ml_metadata.PutParentContextsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.PutParentContextsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactType(); - if (f != null) { - writer.writeMessage( + f = message.getParentContextsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ParentContext.serializeBinaryToWriter ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); + f = message.getTransactionOptions(); if (f != null) { - writer.writeBool( + writer.writeMessage( 2, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); - if (f != null) { - writer.writeBool( - 5, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeBool( - 3, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); - if (f != null) { - writer.writeBool( - 4, - f + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional ArtifactType artifact_type = 1; - * @return {?proto.ml_metadata.ArtifactType} + * repeated ParentContext parent_contexts = 1; + * @return {!Array} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.getArtifactType = function() { - return /** @type{?proto.ml_metadata.ArtifactType} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); +proto.ml_metadata.PutParentContextsRequest.prototype.getParentContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ParentContext, 1)); }; /** - * @param {?proto.ml_metadata.ArtifactType|undefined} value - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.PutParentContextsRequest} returns this */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.setArtifactType = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.PutParentContextsRequest.prototype.setParentContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + * @param {!proto.ml_metadata.ParentContext=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ParentContext} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.clearArtifactType = function() { - return this.setArtifactType(undefined); +proto.ml_metadata.PutParentContextsRequest.prototype.addParentContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ParentContext, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.PutParentContextsRequest} returns this */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.hasArtifactType = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.PutParentContextsRequest.prototype.clearParentContextsList = function() { + return this.setParentContextsList([]); }; /** - * optional bool can_add_fields = 2; - * @return {boolean} + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.getCanAddFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +proto.ml_metadata.PutParentContextsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this - */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.setCanAddFields = function(value) { - return jspb.Message.setField(this, 2, value); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.PutParentContextsRequest} returns this +*/ +proto.ml_metadata.PutParentContextsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.PutParentContextsRequest} returns this */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.clearCanAddFields = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.PutParentContextsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -3542,116 +10207,109 @@ proto.ml_metadata.PutArtifactTypeRequest.prototype.clearCanAddFields = function( * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.hasCanAddFields = function() { +proto.ml_metadata.PutParentContextsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; -/** - * optional bool can_omit_fields = 5; - * @return {boolean} - */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.getCanOmitFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); -}; - - -/** - * @param {boolean} value - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this - */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.setCanOmitFields = function(value) { - return jspb.Message.setField(this, 5, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this - */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.clearCanOmitFields = function() { - return jspb.Message.setField(this, 5, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.hasCanOmitFields = function() { - return jspb.Message.getField(this, 5) != null; -}; - - -/** - * optional bool can_delete_fields = 3; - * @return {boolean} - */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.getCanDeleteFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); -}; - -/** - * @param {boolean} value - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this - */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.setCanDeleteFields = function(value) { - return jspb.Message.setField(this, 3, value); -}; +if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.clearCanDeleteFields = function() { - return jspb.Message.setField(this, 3, undefined); +proto.ml_metadata.PutParentContextsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.PutParentContextsResponse.toObject(opt_includeInstance, this); }; /** - * Returns whether this field is set. - * @return {boolean} + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.PutParentContextsResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.hasCanDeleteFields = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.PutParentContextsResponse.toObject = function(includeInstance, msg) { + var f, obj = { + + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; }; +} /** - * optional bool all_fields_match = 4; - * @return {boolean} + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.PutParentContextsResponse} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.getAllFieldsMatch = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, true)); +proto.ml_metadata.PutParentContextsResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.PutParentContextsResponse; + return proto.ml_metadata.PutParentContextsResponse.deserializeBinaryFromReader(msg, reader); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.PutParentContextsResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.PutParentContextsResponse} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.setAllFieldsMatch = function(value) { - return jspb.Message.setField(this, 4, value); +proto.ml_metadata.PutParentContextsResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + default: + reader.skipField(); + break; + } + } + return msg; }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutArtifactTypeRequest} returns this + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.clearAllFieldsMatch = function() { - return jspb.Message.setField(this, 4, undefined); +proto.ml_metadata.PutParentContextsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.PutParentContextsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * Returns whether this field is set. - * @return {boolean} + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.PutParentContextsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactTypeRequest.prototype.hasAllFieldsMatch = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.PutParentContextsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; }; @@ -3671,8 +10329,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutArtifactTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutArtifactTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByTypeRequest.toObject(opt_includeInstance, this); }; @@ -3681,13 +10339,16 @@ proto.ml_metadata.PutArtifactTypeResponse.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutArtifactTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByTypeRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutArtifactTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByTypeRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -3701,23 +10362,23 @@ proto.ml_metadata.PutArtifactTypeResponse.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutArtifactTypeResponse} + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} */ -proto.ml_metadata.PutArtifactTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutArtifactTypeResponse; - return proto.ml_metadata.PutArtifactTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByTypeRequest; + return proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutArtifactTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByTypeRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutArtifactTypeResponse} + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} */ -proto.ml_metadata.PutArtifactTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -3725,8 +10386,22 @@ proto.ml_metadata.PutArtifactTypeResponse.deserializeBinaryFromReader = function var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setTypeId(value); + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); + break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); + msg.setOptions(value); + break; + case 4: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -3738,59 +10413,192 @@ proto.ml_metadata.PutArtifactTypeResponse.deserializeBinaryFromReader = function /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} - */ -proto.ml_metadata.PutArtifactTypeResponse.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutArtifactTypeResponse.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GetArtifactsByTypeRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GetArtifactsByTypeRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.GetArtifactsByTypeRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( + 1, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } + f = message.getOptions(); + if (f != null) { + writer.writeMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 4, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + +/** + * optional string type_name = 1; + * @return {string} + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasTypeName = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional string type_version = 2; + * @return {string} + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional ListOperationOptions options = 3; + * @return {?proto.ml_metadata.ListOperationOptions} + */ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 3)); +}; + + +/** + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this +*/ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutArtifactTypeResponse} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this */ -proto.ml_metadata.PutArtifactTypeResponse.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( - 1, - f - ); - } +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); }; /** - * optional int64 type_id = 1; - * @return {number} + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.PutArtifactTypeResponse.prototype.getTypeId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * @param {number} value - * @return {!proto.ml_metadata.PutArtifactTypeResponse} returns this + * optional TransactionOptions transaction_options = 4; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutArtifactTypeResponse.prototype.setTypeId = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutArtifactTypeResponse} returns this + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this +*/ +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this */ -proto.ml_metadata.PutArtifactTypeResponse.prototype.clearTypeId = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -3798,8 +10606,8 @@ proto.ml_metadata.PutArtifactTypeResponse.prototype.clearTypeId = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutArtifactTypeResponse.prototype.hasTypeId = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 4) != null; }; @@ -3809,7 +10617,7 @@ proto.ml_metadata.PutArtifactTypeResponse.prototype.hasTypeId = function() { * @private {!Array} * @const */ -proto.ml_metadata.PutExecutionsRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactsByTypeResponse.repeatedFields_ = [1]; @@ -3826,8 +10634,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutExecutionsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByTypeResponse.toObject(opt_includeInstance, this); }; @@ -3836,14 +10644,15 @@ proto.ml_metadata.PutExecutionsRequest.prototype.toObject = function(opt_include * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByTypeResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByTypeResponse.toObject = function(includeInstance, msg) { var f, obj = { - executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), - ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance) + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f }; if (includeInstance) { @@ -3857,23 +10666,23 @@ proto.ml_metadata.PutExecutionsRequest.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionsRequest} + * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} */ -proto.ml_metadata.PutExecutionsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionsRequest; - return proto.ml_metadata.PutExecutionsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByTypeResponse; + return proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByTypeResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionsRequest} + * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} */ -proto.ml_metadata.PutExecutionsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -3881,9 +10690,13 @@ proto.ml_metadata.PutExecutionsRequest.deserializeBinaryFromReader = function(ms var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Execution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); - msg.addExecutions(value); + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setNextPageToken(value); break; default: reader.skipField(); @@ -3898,9 +10711,9 @@ proto.ml_metadata.PutExecutionsRequest.deserializeBinaryFromReader = function(ms * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutExecutionsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByTypeResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -3908,216 +10721,101 @@ proto.ml_metadata.PutExecutionsRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionsRequest} message + * @param {!proto.ml_metadata.GetArtifactsByTypeResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByTypeResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionsList(); + f = message.getArtifactsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f ); } }; /** - * repeated Execution executions = 1; - * @return {!Array} + * repeated Artifact artifacts = 1; + * @return {!Array} */ -proto.ml_metadata.PutExecutionsRequest.prototype.getExecutionsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutExecutionsRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this */ -proto.ml_metadata.PutExecutionsRequest.prototype.setExecutionsList = function(value) { +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.setArtifactsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Execution=} opt_value + * @param {!proto.ml_metadata.Artifact=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Execution} - */ -proto.ml_metadata.PutExecutionsRequest.prototype.addExecutions = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); -}; - - -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutExecutionsRequest} returns this - */ -proto.ml_metadata.PutExecutionsRequest.prototype.clearExecutionsList = function() { - return this.setExecutionsList([]); -}; - - - -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.PutExecutionsResponse.repeatedFields_ = [1]; - - - -if (jspb.Message.GENERATE_TO_OBJECT) { -/** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} - */ -proto.ml_metadata.PutExecutionsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionsResponse.toObject(opt_includeInstance, this); -}; - - -/** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionsResponse} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages - */ -proto.ml_metadata.PutExecutionsResponse.toObject = function(includeInstance, msg) { - var f, obj = { - executionIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; -}; -} - - -/** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionsResponse} - */ -proto.ml_metadata.PutExecutionsResponse.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionsResponse; - return proto.ml_metadata.PutExecutionsResponse.deserializeBinaryFromReader(msg, reader); -}; - - -/** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionsResponse} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionsResponse} - */ -proto.ml_metadata.PutExecutionsResponse.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addExecutionIds(values[i]); - } - break; - default: - reader.skipField(); - break; - } - } - return msg; -}; - - -/** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * @return {!proto.ml_metadata.Artifact} */ -proto.ml_metadata.PutExecutionsResponse.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionsResponse.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionsResponse} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this */ -proto.ml_metadata.PutExecutionsResponse.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getExecutionIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( - 1, - f - ); - } +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); }; /** - * repeated int64 execution_ids = 1; - * @return {!Array} + * optional string next_page_token = 2; + * @return {string} */ -proto.ml_metadata.PutExecutionsResponse.prototype.getExecutionIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutExecutionsResponse} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this */ -proto.ml_metadata.PutExecutionsResponse.prototype.setExecutionIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.PutExecutionsResponse} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this */ -proto.ml_metadata.PutExecutionsResponse.prototype.addExecutionIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 2, undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutExecutionsResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.PutExecutionsResponse.prototype.clearExecutionIdsList = function() { - return this.setExecutionIdsList([]); +proto.ml_metadata.GetArtifactsByTypeResponse.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -4137,8 +10835,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactByTypeAndNameRequest.toObject(opt_includeInstance, this); }; @@ -4147,17 +10845,16 @@ proto.ml_metadata.PutExecutionTypeRequest.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactByTypeAndNameRequest.toObject = function(includeInstance, msg) { var f, obj = { - executionType: (f = msg.getExecutionType()) && ml_metadata_proto_metadata_store_pb.ExecutionType.toObject(includeInstance, f), - canAddFields: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f, - canOmitFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, - canDeleteFields: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, - allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 4, true) + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + artifactName: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -4171,23 +10868,23 @@ proto.ml_metadata.PutExecutionTypeRequest.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionTypeRequest} + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} */ -proto.ml_metadata.PutExecutionTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionTypeRequest; - return proto.ml_metadata.PutExecutionTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactByTypeAndNameRequest; + return proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionTypeRequest} + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} */ -proto.ml_metadata.PutExecutionTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -4195,25 +10892,21 @@ proto.ml_metadata.PutExecutionTypeRequest.deserializeBinaryFromReader = function var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); - msg.setExecutionType(value); - break; - case 2: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanAddFields(value); - break; - case 5: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanOmitFields(value); + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); break; case 3: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanDeleteFields(value); + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setArtifactName(value); break; case 4: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setAllFieldsMatch(value); + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -4228,9 +10921,9 @@ proto.ml_metadata.PutExecutionTypeRequest.deserializeBinaryFromReader = function * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactByTypeAndNameRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -4238,112 +10931,68 @@ proto.ml_metadata.PutExecutionTypeRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionTypeRequest} message + * @param {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactByTypeAndNameRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionType(); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeMessage( + writer.writeString( 1, - f, - ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeBool( - 2, f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + f = /** @type {string} */ (jspb.Message.getField(message, 3)); if (f != null) { - writer.writeBool( - 5, + writer.writeString( + 3, f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); + f = /** @type {string} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeBool( - 3, + writer.writeString( + 2, f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); + f = message.getTransactionOptions(); if (f != null) { - writer.writeBool( + writer.writeMessage( 4, - f + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional ExecutionType execution_type = 1; - * @return {?proto.ml_metadata.ExecutionType} - */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.getExecutionType = function() { - return /** @type{?proto.ml_metadata.ExecutionType} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); -}; - - -/** - * @param {?proto.ml_metadata.ExecutionType|undefined} value - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this -*/ -proto.ml_metadata.PutExecutionTypeRequest.prototype.setExecutionType = function(value) { - return jspb.Message.setWrapperField(this, 1, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this - */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.clearExecutionType = function() { - return this.setExecutionType(undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.hasExecutionType = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * optional bool can_add_fields = 2; - * @return {boolean} + * optional string type_name = 1; + * @return {string} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.getCanAddFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.setCanAddFields = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanAddFields = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -4351,35 +11000,35 @@ proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanAddFields = function * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.hasCanAddFields = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasTypeName = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional bool can_omit_fields = 5; - * @return {boolean} + * optional string type_version = 3; + * @return {string} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.getCanOmitFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.setCanOmitFields = function(value) { - return jspb.Message.setField(this, 5, value); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 3, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanOmitFields = function() { - return jspb.Message.setField(this, 5, undefined); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 3, undefined); }; @@ -4387,35 +11036,35 @@ proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanOmitFields = functio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.hasCanOmitFields = function() { - return jspb.Message.getField(this, 5) != null; +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * optional bool can_delete_fields = 3; - * @return {boolean} + * optional string artifact_name = 2; + * @return {string} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.getCanDeleteFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getArtifactName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.setCanDeleteFields = function(value) { - return jspb.Message.setField(this, 3, value); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setArtifactName = function(value) { + return jspb.Message.setField(this, 2, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanDeleteFields = function() { - return jspb.Message.setField(this, 3, undefined); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearArtifactName = function() { + return jspb.Message.setField(this, 2, undefined); }; @@ -4423,35 +11072,36 @@ proto.ml_metadata.PutExecutionTypeRequest.prototype.clearCanDeleteFields = funct * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.hasCanDeleteFields = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasArtifactName = function() { + return jspb.Message.getField(this, 2) != null; }; /** - * optional bool all_fields_match = 4; - * @return {boolean} + * optional TransactionOptions transaction_options = 4; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.getAllFieldsMatch = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, true)); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this - */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.setAllFieldsMatch = function(value) { - return jspb.Message.setField(this, 4, value); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this +*/ +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutExecutionTypeRequest} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.clearAllFieldsMatch = function() { - return jspb.Message.setField(this, 4, undefined); +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -4459,7 +11109,7 @@ proto.ml_metadata.PutExecutionTypeRequest.prototype.clearAllFieldsMatch = functi * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionTypeRequest.prototype.hasAllFieldsMatch = function() { +proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 4) != null; }; @@ -4480,8 +11130,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutExecutionTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactByTypeAndNameResponse.toObject(opt_includeInstance, this); }; @@ -4490,13 +11140,13 @@ proto.ml_metadata.PutExecutionTypeResponse.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactByTypeAndNameResponse.toObject = function(includeInstance, msg) { var f, obj = { - typeId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + artifact: (f = msg.getArtifact()) && ml_metadata_proto_metadata_store_pb.Artifact.toObject(includeInstance, f) }; if (includeInstance) { @@ -4510,23 +11160,23 @@ proto.ml_metadata.PutExecutionTypeResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionTypeResponse} + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} */ -proto.ml_metadata.PutExecutionTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionTypeResponse; - return proto.ml_metadata.PutExecutionTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactByTypeAndNameResponse; + return proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionTypeResponse} + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} */ -proto.ml_metadata.PutExecutionTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -4534,8 +11184,9 @@ proto.ml_metadata.PutExecutionTypeResponse.deserializeBinaryFromReader = functio var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setTypeId(value); + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.setArtifact(value); break; default: reader.skipField(); @@ -4550,9 +11201,9 @@ proto.ml_metadata.PutExecutionTypeResponse.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutExecutionTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactByTypeAndNameResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -4560,46 +11211,48 @@ proto.ml_metadata.PutExecutionTypeResponse.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionTypeResponse} message + * @param {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactByTypeAndNameResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); + f = message.getArtifact(); if (f != null) { - writer.writeInt64( + writer.writeMessage( 1, - f + f, + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter ); } }; /** - * optional int64 type_id = 1; - * @return {number} + * optional Artifact artifact = 1; + * @return {?proto.ml_metadata.Artifact} */ -proto.ml_metadata.PutExecutionTypeResponse.prototype.getTypeId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.getArtifact = function() { + return /** @type{?proto.ml_metadata.Artifact} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); }; /** - * @param {number} value - * @return {!proto.ml_metadata.PutExecutionTypeResponse} returns this - */ -proto.ml_metadata.PutExecutionTypeResponse.prototype.setTypeId = function(value) { - return jspb.Message.setField(this, 1, value); + * @param {?proto.ml_metadata.Artifact|undefined} value + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} returns this +*/ +proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.setArtifact = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutExecutionTypeResponse} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} returns this */ -proto.ml_metadata.PutExecutionTypeResponse.prototype.clearTypeId = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.clearArtifact = function() { + return this.setArtifact(undefined); }; @@ -4607,7 +11260,7 @@ proto.ml_metadata.PutExecutionTypeResponse.prototype.clearTypeId = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionTypeResponse.prototype.hasTypeId = function() { +proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.hasArtifact = function() { return jspb.Message.getField(this, 1) != null; }; @@ -4618,7 +11271,7 @@ proto.ml_metadata.PutExecutionTypeResponse.prototype.hasTypeId = function() { * @private {!Array} * @const */ -proto.ml_metadata.PutEventsRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactsByIDRequest.repeatedFields_ = [1]; @@ -4635,8 +11288,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutEventsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutEventsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByIDRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByIDRequest.toObject(opt_includeInstance, this); }; @@ -4645,14 +11298,15 @@ proto.ml_metadata.PutEventsRequest.prototype.toObject = function(opt_includeInst * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutEventsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByIDRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutEventsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByIDRequest.toObject = function(includeInstance, msg) { var f, obj = { - eventsList: jspb.Message.toObjectList(msg.getEventsList(), - ml_metadata_proto_metadata_store_pb.Event.toObject, includeInstance) + artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + populateArtifactTypes: jspb.Message.getBooleanFieldWithDefault(msg, 3, false), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -4666,23 +11320,23 @@ proto.ml_metadata.PutEventsRequest.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutEventsRequest} + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} */ -proto.ml_metadata.PutEventsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutEventsRequest; - return proto.ml_metadata.PutEventsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByIDRequest; + return proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutEventsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByIDRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutEventsRequest} + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} */ -proto.ml_metadata.PutEventsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -4690,9 +11344,19 @@ proto.ml_metadata.PutEventsRequest.deserializeBinaryFromReader = function(msg, r var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Event; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); - msg.addEvents(value); + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addArtifactIds(values[i]); + } + break; + case 3: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setPopulateArtifactTypes(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -4707,9 +11371,9 @@ proto.ml_metadata.PutEventsRequest.deserializeBinaryFromReader = function(msg, r * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutEventsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByIDRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutEventsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByIDRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -4717,159 +11381,144 @@ proto.ml_metadata.PutEventsRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutEventsRequest} message + * @param {!proto.ml_metadata.GetArtifactsByIDRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutEventsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByIDRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getEventsList(); + f = message.getArtifactIdsList(); if (f.length > 0) { - writer.writeRepeatedMessage( + writer.writeRepeatedInt64( 1, + f + ); + } + f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeBool( + 3, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 2, f, - ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * repeated Event events = 1; - * @return {!Array} + * repeated int64 artifact_ids = 1; + * @return {!Array} */ -proto.ml_metadata.PutEventsRequest.prototype.getEventsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 1)); +proto.ml_metadata.GetArtifactsByIDRequest.prototype.getArtifactIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutEventsRequest} returns this -*/ -proto.ml_metadata.PutEventsRequest.prototype.setEventsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this + */ +proto.ml_metadata.GetArtifactsByIDRequest.prototype.setArtifactIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * @param {!proto.ml_metadata.Event=} opt_value + * @param {number} value * @param {number=} opt_index - * @return {!proto.ml_metadata.Event} + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this */ -proto.ml_metadata.PutEventsRequest.prototype.addEvents = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Event, opt_index); +proto.ml_metadata.GetArtifactsByIDRequest.prototype.addArtifactIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutEventsRequest} returns this + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this */ -proto.ml_metadata.PutEventsRequest.prototype.clearEventsList = function() { - return this.setEventsList([]); +proto.ml_metadata.GetArtifactsByIDRequest.prototype.clearArtifactIdsList = function() { + return this.setArtifactIdsList([]); }; - - - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * optional bool populate_artifact_types = 3; + * @return {boolean} */ -proto.ml_metadata.PutEventsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutEventsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByIDRequest.prototype.getPopulateArtifactTypes = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutEventsResponse} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * @param {boolean} value + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this */ -proto.ml_metadata.PutEventsResponse.toObject = function(includeInstance, msg) { - var f, obj = { +proto.ml_metadata.GetArtifactsByIDRequest.prototype.setPopulateArtifactTypes = function(value) { + return jspb.Message.setField(this, 3, value); +}; - }; - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this + */ +proto.ml_metadata.GetArtifactsByIDRequest.prototype.clearPopulateArtifactTypes = function() { + return jspb.Message.setField(this, 3, undefined); }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutEventsResponse} + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.PutEventsResponse.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutEventsResponse; - return proto.ml_metadata.PutEventsResponse.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.GetArtifactsByIDRequest.prototype.hasPopulateArtifactTypes = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.PutEventsResponse} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutEventsResponse} + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutEventsResponse.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.GetArtifactsByIDRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this +*/ +proto.ml_metadata.GetArtifactsByIDRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this */ -proto.ml_metadata.PutEventsResponse.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutEventsResponse.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.GetArtifactsByIDRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutEventsResponse} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.PutEventsResponse.serializeBinaryToWriter = function(message, writer) { - var f = undefined; +proto.ml_metadata.GetArtifactsByIDRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -4879,7 +11528,7 @@ proto.ml_metadata.PutEventsResponse.serializeBinaryToWriter = function(message, * @private {!Array} * @const */ -proto.ml_metadata.PutExecutionRequest.repeatedFields_ = [2,3]; +proto.ml_metadata.GetArtifactsByIDResponse.repeatedFields_ = [1,2]; @@ -4896,8 +11545,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutExecutionRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByIDResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByIDResponse.toObject(opt_includeInstance, this); }; @@ -4906,18 +11555,16 @@ proto.ml_metadata.PutExecutionRequest.prototype.toObject = function(opt_includeI * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByIDResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByIDResponse.toObject = function(includeInstance, msg) { var f, obj = { - execution: (f = msg.getExecution()) && ml_metadata_proto_metadata_store_pb.Execution.toObject(includeInstance, f), - artifactEventPairsList: jspb.Message.toObjectList(msg.getArtifactEventPairsList(), - proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.toObject, includeInstance), - contextsList: jspb.Message.toObjectList(msg.getContextsList(), - ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), - options: (f = msg.getOptions()) && proto.ml_metadata.PutExecutionRequest.Options.toObject(includeInstance, f) + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), + artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), + ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance) }; if (includeInstance) { @@ -4931,23 +11578,23 @@ proto.ml_metadata.PutExecutionRequest.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionRequest} + * @return {!proto.ml_metadata.GetArtifactsByIDResponse} */ -proto.ml_metadata.PutExecutionRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionRequest; - return proto.ml_metadata.PutExecutionRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByIDResponse; + return proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByIDResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionRequest} + * @return {!proto.ml_metadata.GetArtifactsByIDResponse} */ -proto.ml_metadata.PutExecutionRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -4955,24 +11602,14 @@ proto.ml_metadata.PutExecutionRequest.deserializeBinaryFromReader = function(msg var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Execution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); - msg.setExecution(value); + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); break; case 2: - var value = new proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent; - reader.readMessage(value,proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinaryFromReader); - msg.addArtifactEventPairs(value); - break; - case 3: - var value = new ml_metadata_proto_metadata_store_pb.Context; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); - msg.addContexts(value); - break; - case 4: - var value = new proto.ml_metadata.PutExecutionRequest.Options; - reader.readMessage(value,proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader); - msg.setOptions(value); + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.addArtifactTypes(value); break; default: reader.skipField(); @@ -4987,9 +11624,9 @@ proto.ml_metadata.PutExecutionRequest.deserializeBinaryFromReader = function(msg * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutExecutionRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByIDResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByIDResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -4997,47 +11634,107 @@ proto.ml_metadata.PutExecutionRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionRequest} message + * @param {!proto.ml_metadata.GetArtifactsByIDResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByIDResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecution(); - if (f != null) { - writer.writeMessage( - 1, - f, - ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter - ); - } - f = message.getArtifactEventPairsList(); + f = message.getArtifactsList(); if (f.length > 0) { writer.writeRepeatedMessage( - 2, + 1, f, - proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter ); } - f = message.getContextsList(); + f = message.getArtifactTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( - 3, - f, - ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter - ); - } - f = message.getOptions(); - if (f != null) { - writer.writeMessage( - 4, + 2, f, - proto.ml_metadata.PutExecutionRequest.Options.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter ); } }; +/** + * repeated Artifact artifacts = 1; + * @return {!Array} + */ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByIDResponse} returns this +*/ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.setArtifactsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Artifact} + */ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetArtifactsByIDResponse} returns this + */ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); +}; + + +/** + * repeated ArtifactType artifact_types = 2; + * @return {!Array} + */ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.getArtifactTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 2)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByIDResponse} returns this +*/ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.setArtifactTypesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 2, value); +}; + + +/** + * @param {!proto.ml_metadata.ArtifactType=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ArtifactType} + */ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.addArtifactTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.ArtifactType, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetArtifactsByIDResponse} returns this + */ +proto.ml_metadata.GetArtifactsByIDResponse.prototype.clearArtifactTypesList = function() { + return this.setArtifactTypesList([]); +}; + + @@ -5054,8 +11751,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsRequest.toObject(opt_includeInstance, this); }; @@ -5064,14 +11761,14 @@ proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.toObject = func * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifact: (f = msg.getArtifact()) && ml_metadata_proto_metadata_store_pb.Artifact.toObject(includeInstance, f), - event: (f = msg.getEvent()) && ml_metadata_proto_metadata_store_pb.Event.toObject(includeInstance, f) + options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -5085,38 +11782,38 @@ proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.toObject = function(inclu /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} + * @return {!proto.ml_metadata.GetArtifactsRequest} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent; - return proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsRequest; + return proto.ml_metadata.GetArtifactsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} + * @return {!proto.ml_metadata.GetArtifactsRequest} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.setArtifact(value); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); + msg.setOptions(value); break; case 2: - var value = new ml_metadata_proto_metadata_store_pb.Event; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); - msg.setEvent(value); + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -5131,9 +11828,9 @@ proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.deserializeBinaryFromRead * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -5141,56 +11838,56 @@ proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.serializeBinary /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} message + * @param {!proto.ml_metadata.GetArtifactsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifact(); + f = message.getOptions(); if (f != null) { writer.writeMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter ); } - f = message.getEvent(); + f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( 2, f, - ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional Artifact artifact = 1; - * @return {?proto.ml_metadata.Artifact} + * optional ListOperationOptions options = 1; + * @return {?proto.ml_metadata.ListOperationOptions} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.getArtifact = function() { - return /** @type{?proto.ml_metadata.Artifact} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.GetArtifactsRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 1)); }; /** - * @param {?proto.ml_metadata.Artifact|undefined} value - * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsRequest} returns this */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.setArtifact = function(value) { +proto.ml_metadata.GetArtifactsRequest.prototype.setOptions = function(value) { return jspb.Message.setWrapperField(this, 1, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this + * @return {!proto.ml_metadata.GetArtifactsRequest} returns this */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.clearArtifact = function() { - return this.setArtifact(undefined); +proto.ml_metadata.GetArtifactsRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); }; @@ -5198,36 +11895,36 @@ proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.clearArtifact = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.hasArtifact = function() { +proto.ml_metadata.GetArtifactsRequest.prototype.hasOptions = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional Event event = 2; - * @return {?proto.ml_metadata.Event} + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.getEvent = function() { - return /** @type{?proto.ml_metadata.Event} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 2)); +proto.ml_metadata.GetArtifactsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** - * @param {?proto.ml_metadata.Event|undefined} value - * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsRequest} returns this */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.setEvent = function(value) { +proto.ml_metadata.GetArtifactsRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} returns this + * @return {!proto.ml_metadata.GetArtifactsRequest} returns this */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.clearEvent = function() { - return this.setEvent(undefined); +proto.ml_metadata.GetArtifactsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -5235,12 +11932,19 @@ proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.clearEvent = fu * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent.prototype.hasEvent = function() { +proto.ml_metadata.GetArtifactsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetArtifactsResponse.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -5256,8 +11960,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutExecutionRequest.Options.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionRequest.Options.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsResponse.toObject(opt_includeInstance, this); }; @@ -5266,13 +11970,15 @@ proto.ml_metadata.PutExecutionRequest.Options.prototype.toObject = function(opt_ * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionRequest.Options} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionRequest.Options.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsResponse.toObject = function(includeInstance, msg) { var f, obj = { - reuseContextIfAlreadyExist: (f = jspb.Message.getBooleanField(msg, 1)) == null ? undefined : f + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f }; if (includeInstance) { @@ -5286,23 +11992,23 @@ proto.ml_metadata.PutExecutionRequest.Options.toObject = function(includeInstanc /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionRequest.Options} + * @return {!proto.ml_metadata.GetArtifactsResponse} */ -proto.ml_metadata.PutExecutionRequest.Options.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionRequest.Options; - return proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsResponse; + return proto.ml_metadata.GetArtifactsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionRequest.Options} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionRequest.Options} + * @return {!proto.ml_metadata.GetArtifactsResponse} */ -proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -5310,8 +12016,13 @@ proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader = func var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setReuseContextIfAlreadyExist(value); + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setNextPageToken(value); break; default: reader.skipField(); @@ -5326,9 +12037,9 @@ proto.ml_metadata.PutExecutionRequest.Options.deserializeBinaryFromReader = func * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutExecutionRequest.Options.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionRequest.Options.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -5336,16 +12047,24 @@ proto.ml_metadata.PutExecutionRequest.Options.prototype.serializeBinary = functi /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionRequest.Options} message + * @param {!proto.ml_metadata.GetArtifactsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionRequest.Options.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {boolean} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeBool( + f = message.getArtifactsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, + f, + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, f ); } @@ -5353,179 +12072,67 @@ proto.ml_metadata.PutExecutionRequest.Options.serializeBinaryToWriter = function /** - * optional bool reuse_context_if_already_exist = 1; - * @return {boolean} - */ -proto.ml_metadata.PutExecutionRequest.Options.prototype.getReuseContextIfAlreadyExist = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 1, false)); -}; - - -/** - * @param {boolean} value - * @return {!proto.ml_metadata.PutExecutionRequest.Options} returns this - */ -proto.ml_metadata.PutExecutionRequest.Options.prototype.setReuseContextIfAlreadyExist = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutExecutionRequest.Options} returns this - */ -proto.ml_metadata.PutExecutionRequest.Options.prototype.clearReuseContextIfAlreadyExist = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.PutExecutionRequest.Options.prototype.hasReuseContextIfAlreadyExist = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * optional Execution execution = 1; - * @return {?proto.ml_metadata.Execution} - */ -proto.ml_metadata.PutExecutionRequest.prototype.getExecution = function() { - return /** @type{?proto.ml_metadata.Execution} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); -}; - - -/** - * @param {?proto.ml_metadata.Execution|undefined} value - * @return {!proto.ml_metadata.PutExecutionRequest} returns this -*/ -proto.ml_metadata.PutExecutionRequest.prototype.setExecution = function(value) { - return jspb.Message.setWrapperField(this, 1, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutExecutionRequest} returns this - */ -proto.ml_metadata.PutExecutionRequest.prototype.clearExecution = function() { - return this.setExecution(undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.PutExecutionRequest.prototype.hasExecution = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * repeated ArtifactAndEvent artifact_event_pairs = 2; - * @return {!Array} - */ -proto.ml_metadata.PutExecutionRequest.prototype.getArtifactEventPairsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent, 2)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.PutExecutionRequest} returns this -*/ -proto.ml_metadata.PutExecutionRequest.prototype.setArtifactEventPairsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 2, value); -}; - - -/** - * @param {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent} - */ -proto.ml_metadata.PutExecutionRequest.prototype.addArtifactEventPairs = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.PutExecutionRequest.ArtifactAndEvent, opt_index); -}; - - -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutExecutionRequest} returns this - */ -proto.ml_metadata.PutExecutionRequest.prototype.clearArtifactEventPairsList = function() { - return this.setArtifactEventPairsList([]); -}; - - -/** - * repeated Context contexts = 3; - * @return {!Array} + * repeated Artifact artifacts = 1; + * @return {!Array} */ -proto.ml_metadata.PutExecutionRequest.prototype.getContextsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 3)); +proto.ml_metadata.GetArtifactsResponse.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutExecutionRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsResponse} returns this */ -proto.ml_metadata.PutExecutionRequest.prototype.setContextsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 3, value); +proto.ml_metadata.GetArtifactsResponse.prototype.setArtifactsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Context=} opt_value + * @param {!proto.ml_metadata.Artifact=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Context} + * @return {!proto.ml_metadata.Artifact} */ -proto.ml_metadata.PutExecutionRequest.prototype.addContexts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 3, opt_value, proto.ml_metadata.Context, opt_index); +proto.ml_metadata.GetArtifactsResponse.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutExecutionRequest} returns this + * @return {!proto.ml_metadata.GetArtifactsResponse} returns this */ -proto.ml_metadata.PutExecutionRequest.prototype.clearContextsList = function() { - return this.setContextsList([]); +proto.ml_metadata.GetArtifactsResponse.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); }; /** - * optional Options options = 4; - * @return {?proto.ml_metadata.PutExecutionRequest.Options} + * optional string next_page_token = 2; + * @return {string} */ -proto.ml_metadata.PutExecutionRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.PutExecutionRequest.Options} */ ( - jspb.Message.getWrapperField(this, proto.ml_metadata.PutExecutionRequest.Options, 4)); +proto.ml_metadata.GetArtifactsResponse.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {?proto.ml_metadata.PutExecutionRequest.Options|undefined} value - * @return {!proto.ml_metadata.PutExecutionRequest} returns this -*/ -proto.ml_metadata.PutExecutionRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactsResponse} returns this + */ +proto.ml_metadata.GetArtifactsResponse.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutExecutionRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsResponse} returns this */ -proto.ml_metadata.PutExecutionRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); +proto.ml_metadata.GetArtifactsResponse.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 2, undefined); }; @@ -5533,8 +12140,8 @@ proto.ml_metadata.PutExecutionRequest.prototype.clearOptions = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutExecutionRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetArtifactsResponse.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -5544,7 +12151,7 @@ proto.ml_metadata.PutExecutionRequest.prototype.hasOptions = function() { * @private {!Array} * @const */ -proto.ml_metadata.PutExecutionResponse.repeatedFields_ = [2,3]; +proto.ml_metadata.GetArtifactsByURIRequest.repeatedFields_ = [2]; @@ -5561,8 +12168,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutExecutionResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutExecutionResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByURIRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByURIRequest.toObject(opt_includeInstance, this); }; @@ -5571,15 +12178,14 @@ proto.ml_metadata.PutExecutionResponse.prototype.toObject = function(opt_include * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutExecutionResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByURIRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByURIRequest.toObject = function(includeInstance, msg) { var f, obj = { - executionId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 2)) == null ? undefined : f, - contextIdsList: (f = jspb.Message.getRepeatedField(msg, 3)) == null ? undefined : f + urisList: (f = jspb.Message.getRepeatedField(msg, 2)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -5593,44 +12199,37 @@ proto.ml_metadata.PutExecutionResponse.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutExecutionResponse} + * @return {!proto.ml_metadata.GetArtifactsByURIRequest} */ -proto.ml_metadata.PutExecutionResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByURIRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutExecutionResponse; - return proto.ml_metadata.PutExecutionResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByURIRequest; + return proto.ml_metadata.GetArtifactsByURIRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutExecutionResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByURIRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutExecutionResponse} + * @return {!proto.ml_metadata.GetArtifactsByURIRequest} */ -proto.ml_metadata.PutExecutionResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByURIRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { - case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setExecutionId(value); - break; case 2: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addArtifactIds(values[i]); - } + var value = /** @type {string} */ (reader.readString()); + msg.addUris(value); break; case 3: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addContextIds(values[i]); - } + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -5645,9 +12244,9 @@ proto.ml_metadata.PutExecutionResponse.deserializeBinaryFromReader = function(ms * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutExecutionResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByURIRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutExecutionResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByURIRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -5655,143 +12254,101 @@ proto.ml_metadata.PutExecutionResponse.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutExecutionResponse} message + * @param {!proto.ml_metadata.GetArtifactsByURIRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutExecutionResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByURIRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( - 1, - f - ); - } - f = message.getArtifactIdsList(); + f = message.getUrisList(); if (f.length > 0) { - writer.writeRepeatedInt64( + writer.writeRepeatedString( 2, f ); } - f = message.getContextIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( 3, - f + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional int64 execution_id = 1; - * @return {number} - */ -proto.ml_metadata.PutExecutionResponse.prototype.getExecutionId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); -}; - - -/** - * @param {number} value - * @return {!proto.ml_metadata.PutExecutionResponse} returns this - */ -proto.ml_metadata.PutExecutionResponse.prototype.setExecutionId = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutExecutionResponse} returns this - */ -proto.ml_metadata.PutExecutionResponse.prototype.clearExecutionId = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.PutExecutionResponse.prototype.hasExecutionId = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * repeated int64 artifact_ids = 2; - * @return {!Array} + * repeated string uris = 2; + * @return {!Array} */ -proto.ml_metadata.PutExecutionResponse.prototype.getArtifactIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 2)); +proto.ml_metadata.GetArtifactsByURIRequest.prototype.getUrisList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 2)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutExecutionResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this */ -proto.ml_metadata.PutExecutionResponse.prototype.setArtifactIdsList = function(value) { +proto.ml_metadata.GetArtifactsByURIRequest.prototype.setUrisList = function(value) { return jspb.Message.setField(this, 2, value || []); }; /** - * @param {number} value + * @param {string} value * @param {number=} opt_index - * @return {!proto.ml_metadata.PutExecutionResponse} returns this + * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this */ -proto.ml_metadata.PutExecutionResponse.prototype.addArtifactIds = function(value, opt_index) { +proto.ml_metadata.GetArtifactsByURIRequest.prototype.addUris = function(value, opt_index) { return jspb.Message.addToRepeatedField(this, 2, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutExecutionResponse} returns this + * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this */ -proto.ml_metadata.PutExecutionResponse.prototype.clearArtifactIdsList = function() { - return this.setArtifactIdsList([]); +proto.ml_metadata.GetArtifactsByURIRequest.prototype.clearUrisList = function() { + return this.setUrisList([]); }; /** - * repeated int64 context_ids = 3; - * @return {!Array} + * optional TransactionOptions transaction_options = 3; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutExecutionResponse.prototype.getContextIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 3)); +proto.ml_metadata.GetArtifactsByURIRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutExecutionResponse} returns this - */ -proto.ml_metadata.PutExecutionResponse.prototype.setContextIdsList = function(value) { - return jspb.Message.setField(this, 3, value || []); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this +*/ +proto.ml_metadata.GetArtifactsByURIRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.PutExecutionResponse} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this */ -proto.ml_metadata.PutExecutionResponse.prototype.addContextIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 3, value, opt_index); +proto.ml_metadata.GetArtifactsByURIRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutExecutionResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.PutExecutionResponse.prototype.clearContextIdsList = function() { - return this.setContextIdsList([]); +proto.ml_metadata.GetArtifactsByURIRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; }; @@ -5801,7 +12358,7 @@ proto.ml_metadata.PutExecutionResponse.prototype.clearContextIdsList = function( * @private {!Array} * @const */ -proto.ml_metadata.PutTypesRequest.repeatedFields_ = [1,2,3]; +proto.ml_metadata.GetArtifactsByURIResponse.repeatedFields_ = [1]; @@ -5818,8 +12375,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutTypesRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutTypesRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByURIResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByURIResponse.toObject(opt_includeInstance, this); }; @@ -5828,22 +12385,14 @@ proto.ml_metadata.PutTypesRequest.prototype.toObject = function(opt_includeInsta * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutTypesRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByURIResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutTypesRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByURIResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), - ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance), - executionTypesList: jspb.Message.toObjectList(msg.getExecutionTypesList(), - ml_metadata_proto_metadata_store_pb.ExecutionType.toObject, includeInstance), - contextTypesList: jspb.Message.toObjectList(msg.getContextTypesList(), - ml_metadata_proto_metadata_store_pb.ContextType.toObject, includeInstance), - canAddFields: (f = jspb.Message.getBooleanField(msg, 4)) == null ? undefined : f, - canOmitFields: (f = jspb.Message.getBooleanField(msg, 7)) == null ? undefined : f, - canDeleteFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, - allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 6, true) + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance) }; if (includeInstance) { @@ -5857,23 +12406,23 @@ proto.ml_metadata.PutTypesRequest.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutTypesRequest} + * @return {!proto.ml_metadata.GetArtifactsByURIResponse} */ -proto.ml_metadata.PutTypesRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByURIResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutTypesRequest; - return proto.ml_metadata.PutTypesRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByURIResponse; + return proto.ml_metadata.GetArtifactsByURIResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutTypesRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByURIResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutTypesRequest} + * @return {!proto.ml_metadata.GetArtifactsByURIResponse} */ -proto.ml_metadata.PutTypesRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByURIResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -5881,35 +12430,9 @@ proto.ml_metadata.PutTypesRequest.deserializeBinaryFromReader = function(msg, re var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); - msg.addArtifactTypes(value); - break; - case 2: - var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); - msg.addExecutionTypes(value); - break; - case 3: - var value = new ml_metadata_proto_metadata_store_pb.ContextType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); - msg.addContextTypes(value); - break; - case 4: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanAddFields(value); - break; - case 7: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanOmitFields(value); - break; - case 5: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanDeleteFields(value); - break; - case 6: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setAllFieldsMatch(value); + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); break; default: reader.skipField(); @@ -5924,9 +12447,9 @@ proto.ml_metadata.PutTypesRequest.deserializeBinaryFromReader = function(msg, re * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutTypesRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByURIResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutTypesRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByURIResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -5934,277 +12457,214 @@ proto.ml_metadata.PutTypesRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutTypesRequest} message + * @param {!proto.ml_metadata.GetArtifactsByURIResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutTypesRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByURIResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactTypesList(); + f = message.getArtifactsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter - ); - } - f = message.getExecutionTypesList(); - if (f.length > 0) { - writer.writeRepeatedMessage( - 2, - f, - ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter - ); - } - f = message.getContextTypesList(); - if (f.length > 0) { - writer.writeRepeatedMessage( - 3, - f, - ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); - if (f != null) { - writer.writeBool( - 4, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 7)); - if (f != null) { - writer.writeBool( - 7, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); - if (f != null) { - writer.writeBool( - 5, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 6)); - if (f != null) { - writer.writeBool( - 6, - f + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter ); } }; /** - * repeated ArtifactType artifact_types = 1; - * @return {!Array} + * repeated Artifact artifacts = 1; + * @return {!Array} */ -proto.ml_metadata.PutTypesRequest.prototype.getArtifactTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); +proto.ml_metadata.GetArtifactsByURIResponse.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutTypesRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByURIResponse} returns this */ -proto.ml_metadata.PutTypesRequest.prototype.setArtifactTypesList = function(value) { +proto.ml_metadata.GetArtifactsByURIResponse.prototype.setArtifactsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ArtifactType=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.ArtifactType} - */ -proto.ml_metadata.PutTypesRequest.prototype.addArtifactTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactType, opt_index); -}; - - -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutTypesRequest} returns this - */ -proto.ml_metadata.PutTypesRequest.prototype.clearArtifactTypesList = function() { - return this.setArtifactTypesList([]); -}; - - -/** - * repeated ExecutionType execution_types = 2; - * @return {!Array} - */ -proto.ml_metadata.PutTypesRequest.prototype.getExecutionTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 2)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.PutTypesRequest} returns this -*/ -proto.ml_metadata.PutTypesRequest.prototype.setExecutionTypesList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 2, value); -}; - - -/** - * @param {!proto.ml_metadata.ExecutionType=} opt_value + * @param {!proto.ml_metadata.Artifact=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.ExecutionType} + * @return {!proto.ml_metadata.Artifact} */ -proto.ml_metadata.PutTypesRequest.prototype.addExecutionTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.ExecutionType, opt_index); +proto.ml_metadata.GetArtifactsByURIResponse.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutTypesRequest} returns this - */ -proto.ml_metadata.PutTypesRequest.prototype.clearExecutionTypesList = function() { - return this.setExecutionTypesList([]); -}; - - -/** - * repeated ContextType context_types = 3; - * @return {!Array} - */ -proto.ml_metadata.PutTypesRequest.prototype.getContextTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 3)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.PutTypesRequest} returns this -*/ -proto.ml_metadata.PutTypesRequest.prototype.setContextTypesList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 3, value); -}; - - -/** - * @param {!proto.ml_metadata.ContextType=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.ContextType} + * @return {!proto.ml_metadata.GetArtifactsByURIResponse} returns this */ -proto.ml_metadata.PutTypesRequest.prototype.addContextTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 3, opt_value, proto.ml_metadata.ContextType, opt_index); +proto.ml_metadata.GetArtifactsByURIResponse.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); }; -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutTypesRequest} returns this - */ -proto.ml_metadata.PutTypesRequest.prototype.clearContextTypesList = function() { - return this.setContextTypesList([]); -}; - -/** - * optional bool can_add_fields = 4; - * @return {boolean} - */ -proto.ml_metadata.PutTypesRequest.prototype.getCanAddFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, false)); -}; +if (jspb.Message.GENERATE_TO_OBJECT) { /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutTypesRequest} returns this + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} */ -proto.ml_metadata.PutTypesRequest.prototype.setCanAddFields = function(value) { - return jspb.Message.setField(this, 4, value); +proto.ml_metadata.GetExecutionsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsRequest.toObject(opt_includeInstance, this); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutTypesRequest} returns this + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.GetExecutionsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutTypesRequest.prototype.clearCanAddFields = function() { - return jspb.Message.setField(this, 4, undefined); -}; - +proto.ml_metadata.GetExecutionsRequest.toObject = function(includeInstance, msg) { + var f, obj = { + options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.PutTypesRequest.prototype.hasCanAddFields = function() { - return jspb.Message.getField(this, 4) != null; + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; }; +} -/** - * optional bool can_omit_fields = 7; - * @return {boolean} +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.GetExecutionsRequest} */ -proto.ml_metadata.PutTypesRequest.prototype.getCanOmitFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 7, false)); +proto.ml_metadata.GetExecutionsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.GetExecutionsRequest; + return proto.ml_metadata.GetExecutionsRequest.deserializeBinaryFromReader(msg, reader); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutTypesRequest} returns this + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.GetExecutionsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.GetExecutionsRequest} */ -proto.ml_metadata.PutTypesRequest.prototype.setCanOmitFields = function(value) { - return jspb.Message.setField(this, 7, value); +proto.ml_metadata.GetExecutionsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); + msg.setOptions(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutTypesRequest} returns this + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.PutTypesRequest.prototype.clearCanOmitFields = function() { - return jspb.Message.setField(this, 7, undefined); +proto.ml_metadata.GetExecutionsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GetExecutionsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * Returns whether this field is set. - * @return {boolean} + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GetExecutionsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutTypesRequest.prototype.hasCanOmitFields = function() { - return jspb.Message.getField(this, 7) != null; +proto.ml_metadata.GetExecutionsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getOptions(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } }; /** - * optional bool can_delete_fields = 5; - * @return {boolean} + * optional ListOperationOptions options = 1; + * @return {?proto.ml_metadata.ListOperationOptions} */ -proto.ml_metadata.PutTypesRequest.prototype.getCanDeleteFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +proto.ml_metadata.GetExecutionsRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 1)); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutTypesRequest} returns this - */ -proto.ml_metadata.PutTypesRequest.prototype.setCanDeleteFields = function(value) { - return jspb.Message.setField(this, 5, value); + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.GetExecutionsRequest} returns this +*/ +proto.ml_metadata.GetExecutionsRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutTypesRequest} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsRequest} returns this */ -proto.ml_metadata.PutTypesRequest.prototype.clearCanDeleteFields = function() { - return jspb.Message.setField(this, 5, undefined); +proto.ml_metadata.GetExecutionsRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); }; @@ -6212,35 +12672,36 @@ proto.ml_metadata.PutTypesRequest.prototype.clearCanDeleteFields = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutTypesRequest.prototype.hasCanDeleteFields = function() { - return jspb.Message.getField(this, 5) != null; +proto.ml_metadata.GetExecutionsRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional bool all_fields_match = 6; - * @return {boolean} + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutTypesRequest.prototype.getAllFieldsMatch = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 6, true)); +proto.ml_metadata.GetExecutionsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutTypesRequest} returns this - */ -proto.ml_metadata.PutTypesRequest.prototype.setAllFieldsMatch = function(value) { - return jspb.Message.setField(this, 6, value); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetExecutionsRequest} returns this +*/ +proto.ml_metadata.GetExecutionsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutTypesRequest} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsRequest} returns this */ -proto.ml_metadata.PutTypesRequest.prototype.clearAllFieldsMatch = function() { - return jspb.Message.setField(this, 6, undefined); +proto.ml_metadata.GetExecutionsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -6248,8 +12709,8 @@ proto.ml_metadata.PutTypesRequest.prototype.clearAllFieldsMatch = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutTypesRequest.prototype.hasAllFieldsMatch = function() { - return jspb.Message.getField(this, 6) != null; +proto.ml_metadata.GetExecutionsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -6259,7 +12720,7 @@ proto.ml_metadata.PutTypesRequest.prototype.hasAllFieldsMatch = function() { * @private {!Array} * @const */ -proto.ml_metadata.PutTypesResponse.repeatedFields_ = [1,2,3]; +proto.ml_metadata.GetExecutionsResponse.repeatedFields_ = [1]; @@ -6276,8 +12737,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutTypesResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutTypesResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsResponse.toObject(opt_includeInstance, this); }; @@ -6286,15 +12747,15 @@ proto.ml_metadata.PutTypesResponse.prototype.toObject = function(opt_includeInst * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutTypesResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutTypesResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactTypeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, - executionTypeIdsList: (f = jspb.Message.getRepeatedField(msg, 2)) == null ? undefined : f, - contextTypeIdsList: (f = jspb.Message.getRepeatedField(msg, 3)) == null ? undefined : f + executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), + ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f }; if (includeInstance) { @@ -6308,23 +12769,23 @@ proto.ml_metadata.PutTypesResponse.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutTypesResponse} + * @return {!proto.ml_metadata.GetExecutionsResponse} */ -proto.ml_metadata.PutTypesResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutTypesResponse; - return proto.ml_metadata.PutTypesResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsResponse; + return proto.ml_metadata.GetExecutionsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutTypesResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutTypesResponse} + * @return {!proto.ml_metadata.GetExecutionsResponse} */ -proto.ml_metadata.PutTypesResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -6332,22 +12793,13 @@ proto.ml_metadata.PutTypesResponse.deserializeBinaryFromReader = function(msg, r var field = reader.getFieldNumber(); switch (field) { case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addArtifactTypeIds(values[i]); - } + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.addExecutions(value); break; case 2: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addExecutionTypeIds(values[i]); - } - break; - case 3: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addContextTypeIds(values[i]); - } + var value = /** @type {string} */ (reader.readString()); + msg.setNextPageToken(value); break; default: reader.skipField(); @@ -6362,154 +12814,111 @@ proto.ml_metadata.PutTypesResponse.deserializeBinaryFromReader = function(msg, r * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutTypesResponse.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutTypesResponse.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); -}; - - -/** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutTypesResponse} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages - */ -proto.ml_metadata.PutTypesResponse.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getArtifactTypeIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( - 1, - f - ); - } - f = message.getExecutionTypeIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( - 2, - f - ); - } - f = message.getContextTypeIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( - 3, - f - ); - } -}; - - -/** - * repeated int64 artifact_type_ids = 1; - * @return {!Array} - */ -proto.ml_metadata.PutTypesResponse.prototype.getArtifactTypeIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.PutTypesResponse} returns this - */ -proto.ml_metadata.PutTypesResponse.prototype.setArtifactTypeIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); -}; - - -/** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.PutTypesResponse} returns this - */ -proto.ml_metadata.PutTypesResponse.prototype.addArtifactTypeIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.GetExecutionsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GetExecutionsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutTypesResponse} returns this + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GetExecutionsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutTypesResponse.prototype.clearArtifactTypeIdsList = function() { - return this.setArtifactTypeIdsList([]); +proto.ml_metadata.GetExecutionsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExecutionsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } }; /** - * repeated int64 execution_type_ids = 2; - * @return {!Array} + * repeated Execution executions = 1; + * @return {!Array} */ -proto.ml_metadata.PutTypesResponse.prototype.getExecutionTypeIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 2)); +proto.ml_metadata.GetExecutionsResponse.prototype.getExecutionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutTypesResponse} returns this - */ -proto.ml_metadata.PutTypesResponse.prototype.setExecutionTypeIdsList = function(value) { - return jspb.Message.setField(this, 2, value || []); + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionsResponse} returns this +*/ +proto.ml_metadata.GetExecutionsResponse.prototype.setExecutionsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {number} value + * @param {!proto.ml_metadata.Execution=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.PutTypesResponse} returns this + * @return {!proto.ml_metadata.Execution} */ -proto.ml_metadata.PutTypesResponse.prototype.addExecutionTypeIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 2, value, opt_index); +proto.ml_metadata.GetExecutionsResponse.prototype.addExecutions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutTypesResponse} returns this + * @return {!proto.ml_metadata.GetExecutionsResponse} returns this */ -proto.ml_metadata.PutTypesResponse.prototype.clearExecutionTypeIdsList = function() { - return this.setExecutionTypeIdsList([]); +proto.ml_metadata.GetExecutionsResponse.prototype.clearExecutionsList = function() { + return this.setExecutionsList([]); }; /** - * repeated int64 context_type_ids = 3; - * @return {!Array} + * optional string next_page_token = 2; + * @return {string} */ -proto.ml_metadata.PutTypesResponse.prototype.getContextTypeIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 3)); +proto.ml_metadata.GetExecutionsResponse.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutTypesResponse} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionsResponse} returns this */ -proto.ml_metadata.PutTypesResponse.prototype.setContextTypeIdsList = function(value) { - return jspb.Message.setField(this, 3, value || []); +proto.ml_metadata.GetExecutionsResponse.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.PutTypesResponse} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsResponse} returns this */ -proto.ml_metadata.PutTypesResponse.prototype.addContextTypeIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 3, value, opt_index); +proto.ml_metadata.GetExecutionsResponse.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 2, undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutTypesResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.PutTypesResponse.prototype.clearContextTypeIdsList = function() { - return this.setContextTypeIdsList([]); +proto.ml_metadata.GetExecutionsResponse.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -6529,8 +12938,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutContextTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutContextTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypeRequest.toObject(opt_includeInstance, this); }; @@ -6539,17 +12948,15 @@ proto.ml_metadata.PutContextTypeRequest.prototype.toObject = function(opt_includ * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutContextTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactTypeRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactTypeRequest.toObject = function(includeInstance, msg) { var f, obj = { - contextType: (f = msg.getContextType()) && ml_metadata_proto_metadata_store_pb.ContextType.toObject(includeInstance, f), - canAddFields: (f = jspb.Message.getBooleanField(msg, 2)) == null ? undefined : f, - canOmitFields: (f = jspb.Message.getBooleanField(msg, 5)) == null ? undefined : f, - canDeleteFields: (f = jspb.Message.getBooleanField(msg, 3)) == null ? undefined : f, - allFieldsMatch: jspb.Message.getBooleanFieldWithDefault(msg, 4, true) + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -6563,23 +12970,23 @@ proto.ml_metadata.PutContextTypeRequest.toObject = function(includeInstance, msg /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutContextTypeRequest} + * @return {!proto.ml_metadata.GetArtifactTypeRequest} */ -proto.ml_metadata.PutContextTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactTypeRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutContextTypeRequest; - return proto.ml_metadata.PutContextTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactTypeRequest; + return proto.ml_metadata.GetArtifactTypeRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutContextTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactTypeRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutContextTypeRequest} + * @return {!proto.ml_metadata.GetArtifactTypeRequest} */ -proto.ml_metadata.PutContextTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactTypeRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -6587,25 +12994,17 @@ proto.ml_metadata.PutContextTypeRequest.deserializeBinaryFromReader = function(m var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ContextType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); - msg.setContextType(value); + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); break; case 2: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanAddFields(value); - break; - case 5: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanOmitFields(value); + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); break; case 3: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setCanDeleteFields(value); - break; - case 4: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setAllFieldsMatch(value); + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -6620,9 +13019,9 @@ proto.ml_metadata.PutContextTypeRequest.deserializeBinaryFromReader = function(m * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutContextTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactTypeRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutContextTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactTypeRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -6630,76 +13029,61 @@ proto.ml_metadata.PutContextTypeRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutContextTypeRequest} message + * @param {!proto.ml_metadata.GetArtifactTypeRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactTypeRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextType(); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeMessage( + writer.writeString( 1, - f, - ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeBool( - 2, f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + f = /** @type {string} */ (jspb.Message.getField(message, 2)); if (f != null) { - writer.writeBool( - 5, + writer.writeString( + 2, f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 3)); + f = message.getTransactionOptions(); if (f != null) { - writer.writeBool( + writer.writeMessage( 3, - f - ); - } - f = /** @type {boolean} */ (jspb.Message.getField(message, 4)); - if (f != null) { - writer.writeBool( - 4, - f + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional ContextType context_type = 1; - * @return {?proto.ml_metadata.ContextType} + * optional string type_name = 1; + * @return {string} */ -proto.ml_metadata.PutContextTypeRequest.prototype.getContextType = function() { - return /** @type{?proto.ml_metadata.ContextType} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); +proto.ml_metadata.GetArtifactTypeRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {?proto.ml_metadata.ContextType|undefined} value - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this -*/ -proto.ml_metadata.PutContextTypeRequest.prototype.setContextType = function(value) { - return jspb.Message.setWrapperField(this, 1, value); + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this + */ +proto.ml_metadata.GetArtifactTypeRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this */ -proto.ml_metadata.PutContextTypeRequest.prototype.clearContextType = function() { - return this.setContextType(undefined); +proto.ml_metadata.GetArtifactTypeRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -6707,143 +13091,223 @@ proto.ml_metadata.PutContextTypeRequest.prototype.clearContextType = function() * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutContextTypeRequest.prototype.hasContextType = function() { +proto.ml_metadata.GetArtifactTypeRequest.prototype.hasTypeName = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional bool can_add_fields = 2; - * @return {boolean} + * optional string type_version = 2; + * @return {string} */ -proto.ml_metadata.PutContextTypeRequest.prototype.getCanAddFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +proto.ml_metadata.GetArtifactTypeRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this */ -proto.ml_metadata.PutContextTypeRequest.prototype.setCanAddFields = function(value) { +proto.ml_metadata.GetArtifactTypeRequest.prototype.setTypeVersion = function(value) { return jspb.Message.setField(this, 2, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this - */ -proto.ml_metadata.PutContextTypeRequest.prototype.clearCanAddFields = function() { - return jspb.Message.setField(this, 2, undefined); + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this + */ +proto.ml_metadata.GetArtifactTypeRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetArtifactTypeRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 3; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.GetArtifactTypeRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this +*/ +proto.ml_metadata.GetArtifactTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this */ -proto.ml_metadata.PutContextTypeRequest.prototype.hasCanAddFields = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetArtifactTypeRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; /** - * optional bool can_omit_fields = 5; + * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutContextTypeRequest.prototype.getCanOmitFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +proto.ml_metadata.GetArtifactTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; }; -/** - * @param {boolean} value - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this - */ -proto.ml_metadata.PutContextTypeRequest.prototype.setCanOmitFields = function(value) { - return jspb.Message.setField(this, 5, value); -}; + +if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} */ -proto.ml_metadata.PutContextTypeRequest.prototype.clearCanOmitFields = function() { - return jspb.Message.setField(this, 5, undefined); +proto.ml_metadata.GetArtifactTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypeResponse.toObject(opt_includeInstance, this); }; /** - * Returns whether this field is set. - * @return {boolean} + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.GetArtifactTypeResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextTypeRequest.prototype.hasCanOmitFields = function() { - return jspb.Message.getField(this, 5) != null; +proto.ml_metadata.GetArtifactTypeResponse.toObject = function(includeInstance, msg) { + var f, obj = { + artifactType: (f = msg.getArtifactType()) && ml_metadata_proto_metadata_store_pb.ArtifactType.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; }; +} /** - * optional bool can_delete_fields = 3; - * @return {boolean} + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.GetArtifactTypeResponse} */ -proto.ml_metadata.PutContextTypeRequest.prototype.getCanDeleteFields = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); +proto.ml_metadata.GetArtifactTypeResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.GetArtifactTypeResponse; + return proto.ml_metadata.GetArtifactTypeResponse.deserializeBinaryFromReader(msg, reader); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.GetArtifactTypeResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.GetArtifactTypeResponse} */ -proto.ml_metadata.PutContextTypeRequest.prototype.setCanDeleteFields = function(value) { - return jspb.Message.setField(this, 3, value); +proto.ml_metadata.GetArtifactTypeResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.setArtifactType(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.PutContextTypeRequest.prototype.clearCanDeleteFields = function() { - return jspb.Message.setField(this, 3, undefined); +proto.ml_metadata.GetArtifactTypeResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GetArtifactTypeResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * Returns whether this field is set. - * @return {boolean} + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GetArtifactTypeResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextTypeRequest.prototype.hasCanDeleteFields = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetArtifactTypeResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactType(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ); + } }; /** - * optional bool all_fields_match = 4; - * @return {boolean} + * optional ArtifactType artifact_type = 1; + * @return {?proto.ml_metadata.ArtifactType} */ -proto.ml_metadata.PutContextTypeRequest.prototype.getAllFieldsMatch = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 4, true)); +proto.ml_metadata.GetArtifactTypeResponse.prototype.getArtifactType = function() { + return /** @type{?proto.ml_metadata.ArtifactType} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); }; /** - * @param {boolean} value - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this - */ -proto.ml_metadata.PutContextTypeRequest.prototype.setAllFieldsMatch = function(value) { - return jspb.Message.setField(this, 4, value); + * @param {?proto.ml_metadata.ArtifactType|undefined} value + * @return {!proto.ml_metadata.GetArtifactTypeResponse} returns this +*/ +proto.ml_metadata.GetArtifactTypeResponse.prototype.setArtifactType = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutContextTypeRequest} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactTypeResponse} returns this */ -proto.ml_metadata.PutContextTypeRequest.prototype.clearAllFieldsMatch = function() { - return jspb.Message.setField(this, 4, undefined); +proto.ml_metadata.GetArtifactTypeResponse.prototype.clearArtifactType = function() { + return this.setArtifactType(undefined); }; @@ -6851,8 +13315,8 @@ proto.ml_metadata.PutContextTypeRequest.prototype.clearAllFieldsMatch = function * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutContextTypeRequest.prototype.hasAllFieldsMatch = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetArtifactTypeResponse.prototype.hasArtifactType = function() { + return jspb.Message.getField(this, 1) != null; }; @@ -6872,8 +13336,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutContextTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutContextTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactTypesRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypesRequest.toObject(opt_includeInstance, this); }; @@ -6882,13 +13346,13 @@ proto.ml_metadata.PutContextTypeResponse.prototype.toObject = function(opt_inclu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutContextTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactTypesRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactTypesRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -6902,23 +13366,23 @@ proto.ml_metadata.PutContextTypeResponse.toObject = function(includeInstance, ms /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutContextTypeResponse} + * @return {!proto.ml_metadata.GetArtifactTypesRequest} */ -proto.ml_metadata.PutContextTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactTypesRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutContextTypeResponse; - return proto.ml_metadata.PutContextTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactTypesRequest; + return proto.ml_metadata.GetArtifactTypesRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutContextTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactTypesRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutContextTypeResponse} + * @return {!proto.ml_metadata.GetArtifactTypesRequest} */ -proto.ml_metadata.PutContextTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactTypesRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -6926,8 +13390,9 @@ proto.ml_metadata.PutContextTypeResponse.deserializeBinaryFromReader = function( var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setTypeId(value); + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -6942,9 +13407,9 @@ proto.ml_metadata.PutContextTypeResponse.deserializeBinaryFromReader = function( * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutContextTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactTypesRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutContextTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactTypesRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -6952,46 +13417,48 @@ proto.ml_metadata.PutContextTypeResponse.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutContextTypeResponse} message + * @param {!proto.ml_metadata.GetArtifactTypesRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactTypesRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); + f = message.getTransactionOptions(); if (f != null) { - writer.writeInt64( + writer.writeMessage( 1, - f + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional int64 type_id = 1; - * @return {number} + * optional TransactionOptions transaction_options = 1; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutContextTypeResponse.prototype.getTypeId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +proto.ml_metadata.GetArtifactTypesRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 1)); }; /** - * @param {number} value - * @return {!proto.ml_metadata.PutContextTypeResponse} returns this - */ -proto.ml_metadata.PutContextTypeResponse.prototype.setTypeId = function(value) { - return jspb.Message.setField(this, 1, value); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactTypesRequest} returns this +*/ +proto.ml_metadata.GetArtifactTypesRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.PutContextTypeResponse} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactTypesRequest} returns this */ -proto.ml_metadata.PutContextTypeResponse.prototype.clearTypeId = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.GetArtifactTypesRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -6999,7 +13466,7 @@ proto.ml_metadata.PutContextTypeResponse.prototype.clearTypeId = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.PutContextTypeResponse.prototype.hasTypeId = function() { +proto.ml_metadata.GetArtifactTypesRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 1) != null; }; @@ -7010,7 +13477,7 @@ proto.ml_metadata.PutContextTypeResponse.prototype.hasTypeId = function() { * @private {!Array} * @const */ -proto.ml_metadata.PutContextsRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactTypesResponse.repeatedFields_ = [1]; @@ -7027,8 +13494,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutContextsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutContextsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactTypesResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypesResponse.toObject(opt_includeInstance, this); }; @@ -7037,14 +13504,14 @@ proto.ml_metadata.PutContextsRequest.prototype.toObject = function(opt_includeIn * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutContextsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactTypesResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactTypesResponse.toObject = function(includeInstance, msg) { var f, obj = { - contextsList: jspb.Message.toObjectList(msg.getContextsList(), - ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) + artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), + ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance) }; if (includeInstance) { @@ -7058,23 +13525,23 @@ proto.ml_metadata.PutContextsRequest.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutContextsRequest} + * @return {!proto.ml_metadata.GetArtifactTypesResponse} */ -proto.ml_metadata.PutContextsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactTypesResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutContextsRequest; - return proto.ml_metadata.PutContextsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactTypesResponse; + return proto.ml_metadata.GetArtifactTypesResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutContextsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactTypesResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutContextsRequest} + * @return {!proto.ml_metadata.GetArtifactTypesResponse} */ -proto.ml_metadata.PutContextsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactTypesResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -7082,9 +13549,9 @@ proto.ml_metadata.PutContextsRequest.deserializeBinaryFromReader = function(msg, var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Context; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); - msg.addContexts(value); + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.addArtifactTypes(value); break; default: reader.skipField(); @@ -7099,9 +13566,9 @@ proto.ml_metadata.PutContextsRequest.deserializeBinaryFromReader = function(msg, * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutContextsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactTypesResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutContextsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactTypesResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -7109,69 +13576,62 @@ proto.ml_metadata.PutContextsRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutContextsRequest} message + * @param {!proto.ml_metadata.GetArtifactTypesResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactTypesResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextsList(); + f = message.getArtifactTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter ); } }; /** - * repeated Context contexts = 1; - * @return {!Array} + * repeated ArtifactType artifact_types = 1; + * @return {!Array} */ -proto.ml_metadata.PutContextsRequest.prototype.getContextsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); +proto.ml_metadata.GetArtifactTypesResponse.prototype.getArtifactTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutContextsRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactTypesResponse} returns this */ -proto.ml_metadata.PutContextsRequest.prototype.setContextsList = function(value) { +proto.ml_metadata.GetArtifactTypesResponse.prototype.setArtifactTypesList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Context=} opt_value + * @param {!proto.ml_metadata.ArtifactType=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Context} + * @return {!proto.ml_metadata.ArtifactType} */ -proto.ml_metadata.PutContextsRequest.prototype.addContexts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); +proto.ml_metadata.GetArtifactTypesResponse.prototype.addArtifactTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactType, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutContextsRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesResponse} returns this */ -proto.ml_metadata.PutContextsRequest.prototype.clearContextsList = function() { - return this.setContextsList([]); +proto.ml_metadata.GetArtifactTypesResponse.prototype.clearArtifactTypesList = function() { + return this.setArtifactTypesList([]); }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.PutContextsResponse.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -7187,8 +13647,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutContextsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutContextsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypesRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypesRequest.toObject(opt_includeInstance, this); }; @@ -7197,13 +13657,13 @@ proto.ml_metadata.PutContextsResponse.prototype.toObject = function(opt_includeI * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutContextsResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypesRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextsResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypesRequest.toObject = function(includeInstance, msg) { var f, obj = { - contextIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -7217,23 +13677,23 @@ proto.ml_metadata.PutContextsResponse.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutContextsResponse} + * @return {!proto.ml_metadata.GetExecutionTypesRequest} */ -proto.ml_metadata.PutContextsResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypesRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutContextsResponse; - return proto.ml_metadata.PutContextsResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypesRequest; + return proto.ml_metadata.GetExecutionTypesRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutContextsResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypesRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutContextsResponse} + * @return {!proto.ml_metadata.GetExecutionTypesRequest} */ -proto.ml_metadata.PutContextsResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypesRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -7241,10 +13701,9 @@ proto.ml_metadata.PutContextsResponse.deserializeBinaryFromReader = function(msg var field = reader.getFieldNumber(); switch (field) { case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addContextIds(values[i]); - } + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -7259,9 +13718,9 @@ proto.ml_metadata.PutContextsResponse.deserializeBinaryFromReader = function(msg * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutContextsResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypesRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutContextsResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypesRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -7269,56 +13728,57 @@ proto.ml_metadata.PutContextsResponse.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutContextsResponse} message + * @param {!proto.ml_metadata.GetExecutionTypesRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutContextsResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypesRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( 1, - f + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * repeated int64 context_ids = 1; - * @return {!Array} + * optional TransactionOptions transaction_options = 1; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.PutContextsResponse.prototype.getContextIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +proto.ml_metadata.GetExecutionTypesRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutContextsResponse} returns this - */ -proto.ml_metadata.PutContextsResponse.prototype.setContextIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetExecutionTypesRequest} returns this +*/ +proto.ml_metadata.GetExecutionTypesRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.PutContextsResponse} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionTypesRequest} returns this */ -proto.ml_metadata.PutContextsResponse.prototype.addContextIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.GetExecutionTypesRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutContextsResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.PutContextsResponse.prototype.clearContextIdsList = function() { - return this.setContextIdsList([]); +proto.ml_metadata.GetExecutionTypesRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 1) != null; }; @@ -7328,7 +13788,7 @@ proto.ml_metadata.PutContextsResponse.prototype.clearContextIdsList = function() * @private {!Array} * @const */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.repeatedFields_ = [1,2]; +proto.ml_metadata.GetExecutionTypesResponse.repeatedFields_ = [1]; @@ -7345,8 +13805,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutAttributionsAndAssociationsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypesResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypesResponse.toObject(opt_includeInstance, this); }; @@ -7355,16 +13815,14 @@ proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.toObject = fun * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypesResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypesResponse.toObject = function(includeInstance, msg) { var f, obj = { - attributionsList: jspb.Message.toObjectList(msg.getAttributionsList(), - ml_metadata_proto_metadata_store_pb.Attribution.toObject, includeInstance), - associationsList: jspb.Message.toObjectList(msg.getAssociationsList(), - ml_metadata_proto_metadata_store_pb.Association.toObject, includeInstance) + executionTypesList: jspb.Message.toObjectList(msg.getExecutionTypesList(), + ml_metadata_proto_metadata_store_pb.ExecutionType.toObject, includeInstance) }; if (includeInstance) { @@ -7378,23 +13836,23 @@ proto.ml_metadata.PutAttributionsAndAssociationsRequest.toObject = function(incl /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} + * @return {!proto.ml_metadata.GetExecutionTypesResponse} */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypesResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutAttributionsAndAssociationsRequest; - return proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypesResponse; + return proto.ml_metadata.GetExecutionTypesResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypesResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} + * @return {!proto.ml_metadata.GetExecutionTypesResponse} */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypesResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -7402,14 +13860,9 @@ proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinaryFromRea var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Attribution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Attribution.deserializeBinaryFromReader); - msg.addAttributions(value); - break; - case 2: - var value = new ml_metadata_proto_metadata_store_pb.Association; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Association.deserializeBinaryFromReader); - msg.addAssociations(value); + var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); + msg.addExecutionTypes(value); break; default: reader.skipField(); @@ -7424,9 +13877,9 @@ proto.ml_metadata.PutAttributionsAndAssociationsRequest.deserializeBinaryFromRea * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypesResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutAttributionsAndAssociationsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypesResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -7434,104 +13887,58 @@ proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.serializeBinar /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} message + * @param {!proto.ml_metadata.GetExecutionTypesResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypesResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getAttributionsList(); - if (f.length > 0) { - writer.writeRepeatedMessage( - 1, - f, - ml_metadata_proto_metadata_store_pb.Attribution.serializeBinaryToWriter - ); - } - f = message.getAssociationsList(); - if (f.length > 0) { - writer.writeRepeatedMessage( - 2, - f, - ml_metadata_proto_metadata_store_pb.Association.serializeBinaryToWriter - ); - } -}; - - -/** - * repeated Attribution attributions = 1; - * @return {!Array} - */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.getAttributionsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Attribution, 1)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this -*/ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.setAttributionsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); -}; - - -/** - * @param {!proto.ml_metadata.Attribution=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.Attribution} - */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.addAttributions = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Attribution, opt_index); -}; - - -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this - */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.clearAttributionsList = function() { - return this.setAttributionsList([]); + f = message.getExecutionTypesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter + ); + } }; /** - * repeated Association associations = 2; - * @return {!Array} + * repeated ExecutionType execution_types = 1; + * @return {!Array} */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.getAssociationsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Association, 2)); +proto.ml_metadata.GetExecutionTypesResponse.prototype.getExecutionTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionTypesResponse} returns this */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.setAssociationsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 2, value); +proto.ml_metadata.GetExecutionTypesResponse.prototype.setExecutionTypesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Association=} opt_value + * @param {!proto.ml_metadata.ExecutionType=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Association} + * @return {!proto.ml_metadata.ExecutionType} */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.addAssociations = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 2, opt_value, proto.ml_metadata.Association, opt_index); +proto.ml_metadata.GetExecutionTypesResponse.prototype.addExecutionTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ExecutionType, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsRequest} returns this + * @return {!proto.ml_metadata.GetExecutionTypesResponse} returns this */ -proto.ml_metadata.PutAttributionsAndAssociationsRequest.prototype.clearAssociationsList = function() { - return this.setAssociationsList([]); +proto.ml_metadata.GetExecutionTypesResponse.prototype.clearExecutionTypesList = function() { + return this.setExecutionTypesList([]); }; @@ -7551,8 +13958,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutAttributionsAndAssociationsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutAttributionsAndAssociationsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypesRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypesRequest.toObject(opt_includeInstance, this); }; @@ -7561,13 +13968,13 @@ proto.ml_metadata.PutAttributionsAndAssociationsResponse.prototype.toObject = fu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypesRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutAttributionsAndAssociationsResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypesRequest.toObject = function(includeInstance, msg) { var f, obj = { - + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -7581,29 +13988,34 @@ proto.ml_metadata.PutAttributionsAndAssociationsResponse.toObject = function(inc /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} + * @return {!proto.ml_metadata.GetContextTypesRequest} */ -proto.ml_metadata.PutAttributionsAndAssociationsResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypesRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutAttributionsAndAssociationsResponse; - return proto.ml_metadata.PutAttributionsAndAssociationsResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypesRequest; + return proto.ml_metadata.GetContextTypesRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypesRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} + * @return {!proto.ml_metadata.GetContextTypesRequest} */ -proto.ml_metadata.PutAttributionsAndAssociationsResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypesRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; default: reader.skipField(); break; @@ -7617,9 +14029,9 @@ proto.ml_metadata.PutAttributionsAndAssociationsResponse.deserializeBinaryFromRe * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutAttributionsAndAssociationsResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypesRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutAttributionsAndAssociationsResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypesRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -7627,12 +14039,57 @@ proto.ml_metadata.PutAttributionsAndAssociationsResponse.prototype.serializeBina /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutAttributionsAndAssociationsResponse} message + * @param {!proto.ml_metadata.GetContextTypesRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutAttributionsAndAssociationsResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypesRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + +/** + * optional TransactionOptions transaction_options = 1; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.GetContextTypesRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 1)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetContextTypesRequest} returns this +*/ +proto.ml_metadata.GetContextTypesRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 1, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetContextTypesRequest} returns this + */ +proto.ml_metadata.GetContextTypesRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetContextTypesRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 1) != null; }; @@ -7642,7 +14099,7 @@ proto.ml_metadata.PutAttributionsAndAssociationsResponse.serializeBinaryToWriter * @private {!Array} * @const */ -proto.ml_metadata.PutParentContextsRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetContextTypesResponse.repeatedFields_ = [1]; @@ -7659,8 +14116,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.PutParentContextsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutParentContextsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypesResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypesResponse.toObject(opt_includeInstance, this); }; @@ -7669,14 +14126,14 @@ proto.ml_metadata.PutParentContextsRequest.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutParentContextsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypesResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutParentContextsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypesResponse.toObject = function(includeInstance, msg) { var f, obj = { - parentContextsList: jspb.Message.toObjectList(msg.getParentContextsList(), - ml_metadata_proto_metadata_store_pb.ParentContext.toObject, includeInstance) + contextTypesList: jspb.Message.toObjectList(msg.getContextTypesList(), + ml_metadata_proto_metadata_store_pb.ContextType.toObject, includeInstance) }; if (includeInstance) { @@ -7690,23 +14147,23 @@ proto.ml_metadata.PutParentContextsRequest.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutParentContextsRequest} + * @return {!proto.ml_metadata.GetContextTypesResponse} */ -proto.ml_metadata.PutParentContextsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypesResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutParentContextsRequest; - return proto.ml_metadata.PutParentContextsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypesResponse; + return proto.ml_metadata.GetContextTypesResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.PutParentContextsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypesResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutParentContextsRequest} + * @return {!proto.ml_metadata.GetContextTypesResponse} */ -proto.ml_metadata.PutParentContextsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypesResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -7714,9 +14171,9 @@ proto.ml_metadata.PutParentContextsRequest.deserializeBinaryFromReader = functio var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ParentContext; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ParentContext.deserializeBinaryFromReader); - msg.addParentContexts(value); + var value = new ml_metadata_proto_metadata_store_pb.ContextType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); + msg.addContextTypes(value); break; default: reader.skipField(); @@ -7731,9 +14188,9 @@ proto.ml_metadata.PutParentContextsRequest.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.PutParentContextsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypesResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutParentContextsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypesResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -7741,162 +14198,68 @@ proto.ml_metadata.PutParentContextsRequest.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutParentContextsRequest} message + * @param {!proto.ml_metadata.GetContextTypesResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.PutParentContextsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypesResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getParentContextsList(); + f = message.getContextTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ParentContext.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter ); } }; /** - * repeated ParentContext parent_contexts = 1; - * @return {!Array} + * repeated ContextType context_types = 1; + * @return {!Array} */ -proto.ml_metadata.PutParentContextsRequest.prototype.getParentContextsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ParentContext, 1)); +proto.ml_metadata.GetContextTypesResponse.prototype.getContextTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.PutParentContextsRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextTypesResponse} returns this */ -proto.ml_metadata.PutParentContextsRequest.prototype.setParentContextsList = function(value) { +proto.ml_metadata.GetContextTypesResponse.prototype.setContextTypesList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ParentContext=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.ParentContext} - */ -proto.ml_metadata.PutParentContextsRequest.prototype.addParentContexts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ParentContext, opt_index); -}; - - -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.PutParentContextsRequest} returns this - */ -proto.ml_metadata.PutParentContextsRequest.prototype.clearParentContextsList = function() { - return this.setParentContextsList([]); -}; - - - - - -if (jspb.Message.GENERATE_TO_OBJECT) { -/** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} - */ -proto.ml_metadata.PutParentContextsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.PutParentContextsResponse.toObject(opt_includeInstance, this); -}; - - -/** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.PutParentContextsResponse} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages - */ -proto.ml_metadata.PutParentContextsResponse.toObject = function(includeInstance, msg) { - var f, obj = { - - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; -}; -} - - -/** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.PutParentContextsResponse} - */ -proto.ml_metadata.PutParentContextsResponse.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.PutParentContextsResponse; - return proto.ml_metadata.PutParentContextsResponse.deserializeBinaryFromReader(msg, reader); -}; - - -/** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.PutParentContextsResponse} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.PutParentContextsResponse} + * @param {!proto.ml_metadata.ContextType=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ContextType} */ -proto.ml_metadata.PutParentContextsResponse.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.GetContextTypesResponse.prototype.addContextTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ContextType, opt_index); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetContextTypesResponse} returns this */ -proto.ml_metadata.PutParentContextsResponse.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.PutParentContextsResponse.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.GetContextTypesResponse.prototype.clearContextTypesList = function() { + return this.setContextTypesList([]); }; + /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.PutParentContextsResponse} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * List of repeated fields within this message type. + * @private {!Array} + * @const */ -proto.ml_metadata.PutParentContextsResponse.serializeBinaryToWriter = function(message, writer) { - var f = undefined; -}; - - +proto.ml_metadata.GetArtifactsByExternalIdsRequest.repeatedFields_ = [1]; @@ -7913,8 +14276,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByExternalIdsRequest.toObject(opt_includeInstance, this); }; @@ -7923,15 +14286,13 @@ proto.ml_metadata.GetArtifactsByTypeRequest.prototype.toObject = function(opt_in * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByExternalIdsRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + externalIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -7946,23 +14307,23 @@ proto.ml_metadata.GetArtifactsByTypeRequest.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByExternalIdsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByTypeRequest; - return proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByExternalIdsRequest; + return proto.ml_metadata.GetArtifactsByExternalIdsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByExternalIdsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -7971,18 +14332,9 @@ proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinaryFromReader = functi switch (field) { case 1: var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); + msg.addExternalIds(value); break; case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); - break; - case 3: - var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); - msg.setOptions(value); - break; - case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -8000,9 +14352,9 @@ proto.ml_metadata.GetArtifactsByTypeRequest.deserializeBinaryFromReader = functi * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByExternalIdsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -8010,38 +14362,23 @@ proto.ml_metadata.GetArtifactsByTypeRequest.prototype.serializeBinary = function /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByTypeRequest} message + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByExternalIdsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( + f = message.getExternalIdsList(); + if (f.length > 0) { + writer.writeRepeatedString( 1, f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } - f = message.getOptions(); - if (f != null) { - writer.writeMessage( - 3, - f, - ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter - ); - } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 4, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -8050,65 +14387,67 @@ proto.ml_metadata.GetArtifactsByTypeRequest.serializeBinaryToWriter = function(m /** - * optional string type_name = 1; - * @return {string} + * repeated string external_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.getExternalIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setTypeName = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.setExternalIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + * @param {string} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearTypeName = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.addExternalIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasTypeName = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.clearExternalIdsList = function() { + return this.setExternalIdsList([]); }; /** - * optional string type_version = 2; - * @return {string} + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this - */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 2, value); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} returns this +*/ +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -8116,82 +14455,168 @@ proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearTypeVersion = functio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasTypeVersion = function() { +proto.ml_metadata.GetArtifactsByExternalIdsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; + /** - * optional ListOperationOptions options = 3; - * @return {?proto.ml_metadata.ListOperationOptions} + * List of repeated fields within this message type. + * @private {!Array} + * @const */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 3)); +proto.ml_metadata.GetArtifactsByExternalIdsResponse.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.GetArtifactsByExternalIdsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByExternalIdsResponse.toObject(opt_includeInstance, this); }; /** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this -*/ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.GetArtifactsByExternalIdsResponse.toObject = function(includeInstance, msg) { + var f, obj = { + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; }; +} /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); +proto.ml_metadata.GetArtifactsByExternalIdsResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.GetArtifactsByExternalIdsResponse; + return proto.ml_metadata.GetArtifactsByExternalIdsResponse.deserializeBinaryFromReader(msg, reader); }; /** - * Returns whether this field is set. - * @return {boolean} + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetArtifactsByExternalIdsResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; }; /** - * optional TransactionOptions transaction_options = 4; - * @return {?proto.ml_metadata.TransactionOptions} + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.getTransactionOptions = function() { - return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); +proto.ml_metadata.GetArtifactsByExternalIdsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GetArtifactsByExternalIdsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GetArtifactsByExternalIdsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.GetArtifactsByExternalIdsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getArtifactsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated Artifact artifacts = 1; + * @return {!Array} + */ +proto.ml_metadata.GetArtifactsByExternalIdsResponse.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); }; /** - * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); +proto.ml_metadata.GetArtifactsByExternalIdsResponse.prototype.setArtifactsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByTypeRequest} returns this + * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Artifact} */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.clearTransactionOptions = function() { - return this.setTransactionOptions(undefined); +proto.ml_metadata.GetArtifactsByExternalIdsResponse.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetArtifactsByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetArtifactsByExternalIdsResponse.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); }; @@ -8201,7 +14626,7 @@ proto.ml_metadata.GetArtifactsByTypeRequest.prototype.hasTransactionOptions = fu * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactsByTypeResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetExecutionsByExternalIdsRequest.repeatedFields_ = [1]; @@ -8218,8 +14643,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByExternalIdsRequest.toObject(opt_includeInstance, this); }; @@ -8228,15 +14653,14 @@ proto.ml_metadata.GetArtifactsByTypeResponse.prototype.toObject = function(opt_i * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByExternalIdsRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), - ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f + externalIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -8250,23 +14674,23 @@ proto.ml_metadata.GetArtifactsByTypeResponse.toObject = function(includeInstance /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByExternalIdsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByTypeResponse; - return proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByExternalIdsRequest; + return proto.ml_metadata.GetExecutionsByExternalIdsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByExternalIdsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -8274,13 +14698,13 @@ proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinaryFromReader = funct var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.addArtifacts(value); + var value = /** @type {string} */ (reader.readString()); + msg.addExternalIds(value); break; case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -8295,9 +14719,9 @@ proto.ml_metadata.GetArtifactsByTypeResponse.deserializeBinaryFromReader = funct * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByExternalIdsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -8305,92 +14729,92 @@ proto.ml_metadata.GetArtifactsByTypeResponse.prototype.serializeBinary = functio /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByTypeResponse} message + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByExternalIdsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactsList(); + f = message.getExternalIdsList(); if (f.length > 0) { - writer.writeRepeatedMessage( + writer.writeRepeatedString( 1, - f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); + f = message.getTransactionOptions(); if (f != null) { - writer.writeString( + writer.writeMessage( 2, - f + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * repeated Artifact artifacts = 1; - * @return {!Array} + * repeated string external_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.getArtifactsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.getExternalIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this -*/ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.setArtifactsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} returns this + */ +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.setExternalIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {string} value * @param {number=} opt_index - * @return {!proto.ml_metadata.Artifact} + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.addArtifacts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.addExternalIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.clearArtifactsList = function() { - return this.setArtifactsList([]); +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.clearExternalIdsList = function() { + return this.setExternalIdsList([]); }; /** - * optional string next_page_token = 2; - * @return {string} + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this - */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 2, value); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} returns this +*/ +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByTypeResponse} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -8398,12 +14822,19 @@ proto.ml_metadata.GetArtifactsByTypeResponse.prototype.clearNextPageToken = func * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByTypeResponse.prototype.hasNextPageToken = function() { +proto.ml_metadata.GetExecutionsByExternalIdsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetExecutionsByExternalIdsResponse.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -8419,8 +14850,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactByTypeAndNameRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByExternalIdsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByExternalIdsResponse.toObject(opt_includeInstance, this); }; @@ -8429,16 +14860,14 @@ proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.toObject = function( * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByExternalIdsResponse.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, - artifactName: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), + ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance) }; if (includeInstance) { @@ -8452,23 +14881,23 @@ proto.ml_metadata.GetArtifactByTypeAndNameRequest.toObject = function(includeIns /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByExternalIdsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactByTypeAndNameRequest; - return proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByExternalIdsResponse; + return proto.ml_metadata.GetExecutionsByExternalIdsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByExternalIdsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -8476,21 +14905,9 @@ proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinaryFromReader = var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); - break; - case 3: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); - break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setArtifactName(value); - break; - case 4: - var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); - msg.setTransactionOptions(value); + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.addExecutions(value); break; default: reader.skipField(); @@ -8505,9 +14922,9 @@ proto.ml_metadata.GetArtifactByTypeAndNameRequest.deserializeBinaryFromReader = * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByExternalIdsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactByTypeAndNameRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByExternalIdsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -8515,176 +14932,255 @@ proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.serializeBinary = fu /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} message + * @param {!proto.ml_metadata.GetExecutionsByExternalIdsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByExternalIdsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( + f = message.getExecutionsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeString( - 3, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } - f = message.getTransactionOptions(); - if (f != null) { - writer.writeMessage( - 4, f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter ); } }; /** - * optional string type_name = 1; - * @return {string} + * repeated Execution executions = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +proto.ml_metadata.GetExecutionsByExternalIdsResponse.prototype.getExecutionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsResponse} returns this +*/ +proto.ml_metadata.GetExecutionsByExternalIdsResponse.prototype.setExecutionsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.ml_metadata.Execution=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Execution} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setTypeName = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.GetExecutionsByExternalIdsResponse.prototype.addExecutions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetExecutionsByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearTypeName = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.GetExecutionsByExternalIdsResponse.prototype.clearExecutionsList = function() { + return this.setExecutionsList([]); }; + /** - * Returns whether this field is set. - * @return {boolean} + * List of repeated fields within this message type. + * @private {!Array} + * @const */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasTypeName = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetContextsByExternalIdsRequest.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByExternalIdsRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.ml_metadata.GetContextsByExternalIdsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.GetContextsByExternalIdsRequest.toObject = function(includeInstance, msg) { + var f, obj = { + externalIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; }; +} /** - * optional string type_version = 3; - * @return {string} + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.ml_metadata.GetContextsByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +proto.ml_metadata.GetContextsByExternalIdsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.ml_metadata.GetContextsByExternalIdsRequest; + return proto.ml_metadata.GetContextsByExternalIdsRequest.deserializeBinaryFromReader(msg, reader); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.ml_metadata.GetContextsByExternalIdsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.ml_metadata.GetContextsByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 3, value); +proto.ml_metadata.GetContextsByExternalIdsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.addExternalIds(value); + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 3, undefined); +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GetContextsByExternalIdsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); }; /** - * Returns whether this field is set. - * @return {boolean} + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GetContextsByExternalIdsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasTypeVersion = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetContextsByExternalIdsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getExternalIdsList(); + if (f.length > 0) { + writer.writeRepeatedString( + 1, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } }; /** - * optional string artifact_name = 2; - * @return {string} + * repeated string external_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getArtifactName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.getExternalIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setArtifactName = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.setExternalIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * @param {string} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetContextsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearArtifactName = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.addExternalIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetContextsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasArtifactName = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.clearExternalIdsList = function() { + return this.setExternalIdsList([]); }; /** - * optional TransactionOptions transaction_options = 4; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * @return {!proto.ml_metadata.GetContextsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameRequest} returns this + * @return {!proto.ml_metadata.GetContextsByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -8693,12 +15189,19 @@ proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.clearTransactionOpti * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactByTypeAndNameRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetContextsByExternalIdsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetContextsByExternalIdsResponse.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -8714,8 +15217,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactByTypeAndNameResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByExternalIdsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByExternalIdsResponse.toObject(opt_includeInstance, this); }; @@ -8724,13 +15227,14 @@ proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.toObject = function * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByExternalIdsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByExternalIdsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifact: (f = msg.getArtifact()) && ml_metadata_proto_metadata_store_pb.Artifact.toObject(includeInstance, f) + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) }; if (includeInstance) { @@ -8744,23 +15248,23 @@ proto.ml_metadata.GetArtifactByTypeAndNameResponse.toObject = function(includeIn /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} + * @return {!proto.ml_metadata.GetContextsByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByExternalIdsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactByTypeAndNameResponse; - return proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByExternalIdsResponse; + return proto.ml_metadata.GetContextsByExternalIdsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByExternalIdsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} + * @return {!proto.ml_metadata.GetContextsByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByExternalIdsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -8768,9 +15272,9 @@ proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinaryFromReader = var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.setArtifact(value); + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addContexts(value); break; default: reader.skipField(); @@ -8785,9 +15289,9 @@ proto.ml_metadata.GetArtifactByTypeAndNameResponse.deserializeBinaryFromReader = * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByExternalIdsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactByTypeAndNameResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByExternalIdsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -8795,57 +15299,58 @@ proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.serializeBinary = f /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} message + * @param {!proto.ml_metadata.GetContextsByExternalIdsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByExternalIdsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifact(); - if (f != null) { - writer.writeMessage( + f = message.getContextsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter ); } }; /** - * optional Artifact artifact = 1; - * @return {?proto.ml_metadata.Artifact} + * repeated Context contexts = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.getArtifact = function() { - return /** @type{?proto.ml_metadata.Artifact} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.GetContextsByExternalIdsResponse.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * @param {?proto.ml_metadata.Artifact|undefined} value - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextsByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.setArtifact = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.GetContextsByExternalIdsResponse.prototype.setContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactByTypeAndNameResponse} returns this + * @param {!proto.ml_metadata.Context=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.clearArtifact = function() { - return this.setArtifact(undefined); +proto.ml_metadata.GetContextsByExternalIdsResponse.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetContextsByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.hasArtifact = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetContextsByExternalIdsResponse.prototype.clearContextsList = function() { + return this.setContextsList([]); }; @@ -8855,7 +15360,7 @@ proto.ml_metadata.GetArtifactByTypeAndNameResponse.prototype.hasArtifact = funct * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactsByIDRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.repeatedFields_ = [1]; @@ -8872,8 +15377,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByIDRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.toObject(opt_includeInstance, this); }; @@ -8882,13 +15387,13 @@ proto.ml_metadata.GetArtifactsByIDRequest.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByIDRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByIDRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + externalIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -8903,23 +15408,23 @@ proto.ml_metadata.GetArtifactsByIDRequest.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByIDRequest} + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByIDRequest; - return proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactTypesByExternalIdsRequest; + return proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByIDRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByIDRequest} + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -8927,10 +15432,8 @@ proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinaryFromReader = function var field = reader.getFieldNumber(); switch (field) { case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addArtifactIds(values[i]); - } + var value = /** @type {string} */ (reader.readString()); + msg.addExternalIds(value); break; case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; @@ -8950,9 +15453,9 @@ proto.ml_metadata.GetArtifactsByIDRequest.deserializeBinaryFromReader = function * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByIDRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -8960,15 +15463,15 @@ proto.ml_metadata.GetArtifactsByIDRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByIDRequest} message + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByIDRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactIdsList(); + f = message.getExternalIdsList(); if (f.length > 0) { - writer.writeRepeatedInt64( + writer.writeRepeatedString( 1, f ); @@ -8985,39 +15488,39 @@ proto.ml_metadata.GetArtifactsByIDRequest.serializeBinaryToWriter = function(mes /** - * repeated int64 artifact_ids = 1; - * @return {!Array} + * repeated string external_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.getArtifactIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.getExternalIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.setArtifactIdsList = function(value) { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.setExternalIdsList = function(value) { return jspb.Message.setField(this, 1, value || []); }; /** - * @param {number} value + * @param {string} value * @param {number=} opt_index - * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.addArtifactIds = function(value, opt_index) { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.addExternalIds = function(value, opt_index) { return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.clearArtifactIdsList = function() { - return this.setArtifactIdsList([]); +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.clearExternalIdsList = function() { + return this.setExternalIdsList([]); }; @@ -9025,7 +15528,7 @@ proto.ml_metadata.GetArtifactsByIDRequest.prototype.clearArtifactIdsList = funct * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -9033,18 +15536,18 @@ proto.ml_metadata.GetArtifactsByIDRequest.prototype.getTransactionOptions = func /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -9053,7 +15556,7 @@ proto.ml_metadata.GetArtifactsByIDRequest.prototype.clearTransactionOptions = fu * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByIDRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetArtifactTypesByExternalIdsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -9064,7 +15567,7 @@ proto.ml_metadata.GetArtifactsByIDRequest.prototype.hasTransactionOptions = func * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactsByIDResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.repeatedFields_ = [1]; @@ -9081,8 +15584,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByIDResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByIDResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.toObject(opt_includeInstance, this); }; @@ -9091,14 +15594,14 @@ proto.ml_metadata.GetArtifactsByIDResponse.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByIDResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByIDResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), - ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance) + artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), + ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance) }; if (includeInstance) { @@ -9112,23 +15615,23 @@ proto.ml_metadata.GetArtifactsByIDResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByIDResponse} + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByIDResponse; - return proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactTypesByExternalIdsResponse; + return proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByIDResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByIDResponse} + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -9136,9 +15639,9 @@ proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinaryFromReader = functio var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.addArtifacts(value); + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.addArtifactTypes(value); break; default: reader.skipField(); @@ -9153,9 +15656,9 @@ proto.ml_metadata.GetArtifactsByIDResponse.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByIDResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByIDResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -9163,62 +15666,69 @@ proto.ml_metadata.GetArtifactsByIDResponse.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByIDResponse} message + * @param {!proto.ml_metadata.GetArtifactTypesByExternalIdsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByIDResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactsList(); + f = message.getArtifactTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter ); } }; /** - * repeated Artifact artifacts = 1; - * @return {!Array} + * repeated ArtifactType artifact_types = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsByIDResponse.prototype.getArtifactsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.prototype.getArtifactTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactsByIDResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsByIDResponse.prototype.setArtifactsList = function(value) { +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.prototype.setArtifactTypesList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {!proto.ml_metadata.ArtifactType=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Artifact} + * @return {!proto.ml_metadata.ArtifactType} */ -proto.ml_metadata.GetArtifactsByIDResponse.prototype.addArtifacts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.prototype.addArtifactTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactType, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactsByIDResponse} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsByIDResponse.prototype.clearArtifactsList = function() { - return this.setArtifactsList([]); +proto.ml_metadata.GetArtifactTypesByExternalIdsResponse.prototype.clearArtifactTypesList = function() { + return this.setArtifactTypesList([]); }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -9234,8 +15744,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.toObject(opt_includeInstance, this); }; @@ -9244,13 +15754,13 @@ proto.ml_metadata.GetArtifactsRequest.prototype.toObject = function(opt_includeI * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.toObject = function(includeInstance, msg) { var f, obj = { - options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + externalIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -9265,23 +15775,23 @@ proto.ml_metadata.GetArtifactsRequest.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsRequest} + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsRequest; - return proto.ml_metadata.GetArtifactsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypesByExternalIdsRequest; + return proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsRequest} + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -9289,9 +15799,8 @@ proto.ml_metadata.GetArtifactsRequest.deserializeBinaryFromReader = function(msg var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); - msg.setOptions(value); + var value = /** @type {string} */ (reader.readString()); + msg.addExternalIds(value); break; case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; @@ -9311,9 +15820,9 @@ proto.ml_metadata.GetArtifactsRequest.deserializeBinaryFromReader = function(msg * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -9321,18 +15830,17 @@ proto.ml_metadata.GetArtifactsRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsRequest} message + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getOptions(); - if (f != null) { - writer.writeMessage( + f = message.getExternalIdsList(); + if (f.length > 0) { + writer.writeRepeatedString( 1, - f, - ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter + f ); } f = message.getTransactionOptions(); @@ -9347,39 +15855,39 @@ proto.ml_metadata.GetArtifactsRequest.serializeBinaryToWriter = function(message /** - * optional ListOperationOptions options = 1; - * @return {?proto.ml_metadata.ListOperationOptions} + * repeated string external_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 1)); +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.getExternalIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsRequest} returns this -*/ -proto.ml_metadata.GetArtifactsRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 1, value); + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} returns this + */ +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.setExternalIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsRequest} returns this + * @param {string} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.addExternalIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.clearExternalIdsList = function() { + return this.setExternalIdsList([]); }; @@ -9387,7 +15895,7 @@ proto.ml_metadata.GetArtifactsRequest.prototype.hasOptions = function() { * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactsRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -9395,18 +15903,18 @@ proto.ml_metadata.GetArtifactsRequest.prototype.getTransactionOptions = function /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsRequest} returns this + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsRequest} returns this + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -9415,7 +15923,7 @@ proto.ml_metadata.GetArtifactsRequest.prototype.clearTransactionOptions = functi * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetExecutionTypesByExternalIdsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -9426,7 +15934,7 @@ proto.ml_metadata.GetArtifactsRequest.prototype.hasTransactionOptions = function * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactsResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.repeatedFields_ = [1]; @@ -9443,8 +15951,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.toObject(opt_includeInstance, this); }; @@ -9453,15 +15961,14 @@ proto.ml_metadata.GetArtifactsResponse.prototype.toObject = function(opt_include * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), - ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f + executionTypesList: jspb.Message.toObjectList(msg.getExecutionTypesList(), + ml_metadata_proto_metadata_store_pb.ExecutionType.toObject, includeInstance) }; if (includeInstance) { @@ -9475,23 +15982,23 @@ proto.ml_metadata.GetArtifactsResponse.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsResponse} + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsResponse; - return proto.ml_metadata.GetArtifactsResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypesByExternalIdsResponse; + return proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsResponse} + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -9499,13 +16006,9 @@ proto.ml_metadata.GetArtifactsResponse.deserializeBinaryFromReader = function(ms var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.addArtifacts(value); - break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); + var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); + msg.addExecutionTypes(value); break; default: reader.skipField(); @@ -9520,9 +16023,9 @@ proto.ml_metadata.GetArtifactsResponse.deserializeBinaryFromReader = function(ms * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -9530,101 +16033,58 @@ proto.ml_metadata.GetArtifactsResponse.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsResponse} message + * @param {!proto.ml_metadata.GetExecutionTypesByExternalIdsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactsList(); + f = message.getExecutionTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f + ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter ); } }; /** - * repeated Artifact artifacts = 1; - * @return {!Array} + * repeated ExecutionType execution_types = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsResponse.prototype.getArtifactsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.prototype.getExecutionTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactsResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsResponse.prototype.setArtifactsList = function(value) { +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.prototype.setExecutionTypesList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {!proto.ml_metadata.ExecutionType=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Artifact} + * @return {!proto.ml_metadata.ExecutionType} */ -proto.ml_metadata.GetArtifactsResponse.prototype.addArtifacts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.prototype.addExecutionTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ExecutionType, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactsResponse} returns this - */ -proto.ml_metadata.GetArtifactsResponse.prototype.clearArtifactsList = function() { - return this.setArtifactsList([]); -}; - - -/** - * optional string next_page_token = 2; - * @return {string} - */ -proto.ml_metadata.GetArtifactsResponse.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactsResponse} returns this + * @return {!proto.ml_metadata.GetExecutionTypesByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsResponse.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 2, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsResponse} returns this - */ -proto.ml_metadata.GetArtifactsResponse.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 2, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetArtifactsResponse.prototype.hasNextPageToken = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetExecutionTypesByExternalIdsResponse.prototype.clearExecutionTypesList = function() { + return this.setExecutionTypesList([]); }; @@ -9634,7 +16094,7 @@ proto.ml_metadata.GetArtifactsResponse.prototype.hasNextPageToken = function() { * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactsByURIRequest.repeatedFields_ = [2]; +proto.ml_metadata.GetContextTypesByExternalIdsRequest.repeatedFields_ = [1]; @@ -9651,8 +16111,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByURIRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypesByExternalIdsRequest.toObject(opt_includeInstance, this); }; @@ -9661,13 +16121,13 @@ proto.ml_metadata.GetArtifactsByURIRequest.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByURIRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByURIRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypesByExternalIdsRequest.toObject = function(includeInstance, msg) { var f, obj = { - urisList: (f = jspb.Message.getRepeatedField(msg, 2)) == null ? undefined : f, + externalIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -9682,34 +16142,34 @@ proto.ml_metadata.GetArtifactsByURIRequest.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByURIRequest} + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByURIRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypesByExternalIdsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByURIRequest; - return proto.ml_metadata.GetArtifactsByURIRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypesByExternalIdsRequest; + return proto.ml_metadata.GetContextTypesByExternalIdsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByURIRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByURIRequest} + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} */ -proto.ml_metadata.GetArtifactsByURIRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypesByExternalIdsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { - case 2: + case 1: var value = /** @type {string} */ (reader.readString()); - msg.addUris(value); + msg.addExternalIds(value); break; - case 3: + case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -9727,9 +16187,9 @@ proto.ml_metadata.GetArtifactsByURIRequest.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByURIRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypesByExternalIdsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -9737,23 +16197,23 @@ proto.ml_metadata.GetArtifactsByURIRequest.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByURIRequest} message + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByURIRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypesByExternalIdsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getUrisList(); + f = message.getExternalIdsList(); if (f.length > 0) { writer.writeRepeatedString( - 2, + 1, f ); } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 3, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -9762,66 +16222,66 @@ proto.ml_metadata.GetArtifactsByURIRequest.serializeBinaryToWriter = function(me /** - * repeated string uris = 2; + * repeated string external_ids = 1; * @return {!Array} */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.getUrisList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 2)); +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.getExternalIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.setUrisList = function(value) { - return jspb.Message.setField(this, 2, value || []); +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.setExternalIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** * @param {string} value * @param {number=} opt_index - * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.addUris = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 2, value, opt_index); +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.addExternalIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.clearUrisList = function() { - return this.setUrisList([]); +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.clearExternalIdsList = function() { + return this.setExternalIdsList([]); }; /** - * optional TransactionOptions transaction_options = 3; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByURIRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsRequest} returns this */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -9830,8 +16290,8 @@ proto.ml_metadata.GetArtifactsByURIRequest.prototype.clearTransactionOptions = f * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByURIRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetContextTypesByExternalIdsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -9841,7 +16301,7 @@ proto.ml_metadata.GetArtifactsByURIRequest.prototype.hasTransactionOptions = fun * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactsByURIResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetContextTypesByExternalIdsResponse.repeatedFields_ = [1]; @@ -9858,8 +16318,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByURIResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByURIResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypesByExternalIdsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypesByExternalIdsResponse.toObject(opt_includeInstance, this); }; @@ -9868,14 +16328,14 @@ proto.ml_metadata.GetArtifactsByURIResponse.prototype.toObject = function(opt_in * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByURIResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByURIResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypesByExternalIdsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), - ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance) + contextTypesList: jspb.Message.toObjectList(msg.getContextTypesList(), + ml_metadata_proto_metadata_store_pb.ContextType.toObject, includeInstance) }; if (includeInstance) { @@ -9889,33 +16349,33 @@ proto.ml_metadata.GetArtifactsByURIResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByURIResponse} + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsByURIResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypesByExternalIdsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByURIResponse; - return proto.ml_metadata.GetArtifactsByURIResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypesByExternalIdsResponse; + return proto.ml_metadata.GetContextTypesByExternalIdsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByURIResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByURIResponse} + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsResponse} */ -proto.ml_metadata.GetArtifactsByURIResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypesByExternalIdsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { - case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.addArtifacts(value); + case 1: + var value = new ml_metadata_proto_metadata_store_pb.ContextType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); + msg.addContextTypes(value); break; default: reader.skipField(); @@ -9930,9 +16390,9 @@ proto.ml_metadata.GetArtifactsByURIResponse.deserializeBinaryFromReader = functi * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByURIResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypesByExternalIdsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByURIResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypesByExternalIdsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -9940,58 +16400,58 @@ proto.ml_metadata.GetArtifactsByURIResponse.prototype.serializeBinary = function /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByURIResponse} message + * @param {!proto.ml_metadata.GetContextTypesByExternalIdsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByURIResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypesByExternalIdsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactsList(); + f = message.getContextTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter ); } }; /** - * repeated Artifact artifacts = 1; - * @return {!Array} + * repeated ContextType context_types = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsByURIResponse.prototype.getArtifactsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.GetContextTypesByExternalIdsResponse.prototype.getContextTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactsByURIResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsByURIResponse.prototype.setArtifactsList = function(value) { +proto.ml_metadata.GetContextTypesByExternalIdsResponse.prototype.setContextTypesList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {!proto.ml_metadata.ContextType=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Artifact} + * @return {!proto.ml_metadata.ContextType} */ -proto.ml_metadata.GetArtifactsByURIResponse.prototype.addArtifacts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +proto.ml_metadata.GetContextTypesByExternalIdsResponse.prototype.addContextTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ContextType, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactsByURIResponse} returns this + * @return {!proto.ml_metadata.GetContextTypesByExternalIdsResponse} returns this */ -proto.ml_metadata.GetArtifactsByURIResponse.prototype.clearArtifactsList = function() { - return this.setArtifactsList([]); +proto.ml_metadata.GetContextTypesByExternalIdsResponse.prototype.clearContextTypesList = function() { + return this.setContextTypesList([]); }; @@ -10011,8 +16471,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByTypeRequest.toObject(opt_includeInstance, this); }; @@ -10021,12 +16481,14 @@ proto.ml_metadata.GetExecutionsRequest.prototype.toObject = function(opt_include * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByTypeRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByTypeRequest.toObject = function(includeInstance, msg) { var f, obj = { + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -10042,23 +16504,23 @@ proto.ml_metadata.GetExecutionsRequest.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsRequest} + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} */ -proto.ml_metadata.GetExecutionsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsRequest; - return proto.ml_metadata.GetExecutionsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByTypeRequest; + return proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByTypeRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsRequest} + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} */ -proto.ml_metadata.GetExecutionsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -10066,11 +16528,19 @@ proto.ml_metadata.GetExecutionsRequest.deserializeBinaryFromReader = function(ms var field = reader.getFieldNumber(); switch (field) { case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); + break; + case 3: var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); msg.setOptions(value); break; - case 2: + case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -10088,9 +16558,9 @@ proto.ml_metadata.GetExecutionsRequest.deserializeBinaryFromReader = function(ms * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByTypeRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -10098,16 +16568,30 @@ proto.ml_metadata.GetExecutionsRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsRequest} message + * @param {!proto.ml_metadata.GetExecutionsByTypeRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByTypeRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( + 1, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } f = message.getOptions(); if (f != null) { writer.writeMessage( - 1, + 3, f, ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter ); @@ -10115,7 +16599,7 @@ proto.ml_metadata.GetExecutionsRequest.serializeBinaryToWriter = function(messag f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 2, + 4, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -10124,67 +16608,29 @@ proto.ml_metadata.GetExecutionsRequest.serializeBinaryToWriter = function(messag /** - * optional ListOperationOptions options = 1; - * @return {?proto.ml_metadata.ListOperationOptions} - */ -proto.ml_metadata.GetExecutionsRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 1)); -}; - - -/** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsRequest} returns this -*/ -proto.ml_metadata.GetExecutionsRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 1, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsRequest} returns this - */ -proto.ml_metadata.GetExecutionsRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} + * optional string type_name = 1; + * @return {string} */ -proto.ml_metadata.GetExecutionsRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * optional TransactionOptions transaction_options = 2; - * @return {?proto.ml_metadata.TransactionOptions} + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionsRequest.prototype.getTransactionOptions = function() { - return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); -}; - - -/** - * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsRequest} returns this -*/ -proto.ml_metadata.GetExecutionsRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionsRequest.prototype.clearTransactionOptions = function() { - return this.setTransactionOptions(undefined); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -10192,207 +16638,109 @@ proto.ml_metadata.GetExecutionsRequest.prototype.clearTransactionOptions = funct * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionsRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 2) != null; -}; - - - -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetExecutionsResponse.repeatedFields_ = [1]; - - - -if (jspb.Message.GENERATE_TO_OBJECT) { -/** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} - */ -proto.ml_metadata.GetExecutionsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsResponse.toObject(opt_includeInstance, this); -}; - - -/** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsResponse} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages - */ -proto.ml_metadata.GetExecutionsResponse.toObject = function(includeInstance, msg) { - var f, obj = { - executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), - ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasTypeName = function() { + return jspb.Message.getField(this, 1) != null; }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsResponse} + * optional string type_version = 2; + * @return {string} */ -proto.ml_metadata.GetExecutionsResponse.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsResponse; - return proto.ml_metadata.GetExecutionsResponse.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; - - -/** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsResponse} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsResponse} - */ -proto.ml_metadata.GetExecutionsResponse.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = new ml_metadata_proto_metadata_store_pb.Execution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); - msg.addExecutions(value); - break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); - break; - default: - reader.skipField(); - break; - } - } - return msg; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this + */ +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionsResponse.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsResponse.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 2, undefined); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsResponse} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetExecutionsResponse.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getExecutionsList(); - if (f.length > 0) { - writer.writeRepeatedMessage( - 1, - f, - ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 2) != null; }; /** - * repeated Execution executions = 1; - * @return {!Array} + * optional ListOperationOptions options = 3; + * @return {?proto.ml_metadata.ListOperationOptions} */ -proto.ml_metadata.GetExecutionsResponse.prototype.getExecutionsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 3)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionsResponse} returns this + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionsResponse.prototype.setExecutionsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** - * @param {!proto.ml_metadata.Execution=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.Execution} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionsResponse.prototype.addExecutions = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionsResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetExecutionsResponse.prototype.clearExecutionsList = function() { - return this.setExecutionsList([]); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * optional string next_page_token = 2; - * @return {string} + * optional TransactionOptions transaction_options = 4; + * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionsResponse.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionsResponse} returns this - */ -proto.ml_metadata.GetExecutionsResponse.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 2, value); + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this +*/ +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsResponse} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionsResponse.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; @@ -10400,12 +16748,19 @@ proto.ml_metadata.GetExecutionsResponse.prototype.clearNextPageToken = function( * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionsResponse.prototype.hasNextPageToken = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 4) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetExecutionsByTypeResponse.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -10421,8 +16776,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByTypeResponse.toObject(opt_includeInstance, this); }; @@ -10431,15 +16786,15 @@ proto.ml_metadata.GetArtifactTypeRequest.prototype.toObject = function(opt_inclu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByTypeResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByTypeResponse.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), + ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f }; if (includeInstance) { @@ -10453,23 +16808,23 @@ proto.ml_metadata.GetArtifactTypeRequest.toObject = function(includeInstance, ms /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactTypeRequest} + * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} */ -proto.ml_metadata.GetArtifactTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactTypeRequest; - return proto.ml_metadata.GetArtifactTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByTypeResponse; + return proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByTypeResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactTypeRequest} + * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} */ -proto.ml_metadata.GetArtifactTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -10477,17 +16832,13 @@ proto.ml_metadata.GetArtifactTypeRequest.deserializeBinaryFromReader = function( var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.addExecutions(value); break; case 2: var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); - break; - case 3: - var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); - msg.setTransactionOptions(value); + msg.setNextPageToken(value); break; default: reader.skipField(); @@ -10502,9 +16853,9 @@ proto.ml_metadata.GetArtifactTypeRequest.deserializeBinaryFromReader = function( * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByTypeResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -10512,17 +16863,18 @@ proto.ml_metadata.GetArtifactTypeRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactTypeRequest} message + * @param {!proto.ml_metadata.GetExecutionsByTypeResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByTypeResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( + f = message.getExecutionsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, - f + f, + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter ); } f = /** @type {string} */ (jspb.Message.getField(message, 2)); @@ -10532,114 +16884,71 @@ proto.ml_metadata.GetArtifactTypeRequest.serializeBinaryToWriter = function(mess f ); } - f = message.getTransactionOptions(); - if (f != null) { - writer.writeMessage( - 3, - f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter - ); - } -}; - - -/** - * optional string type_name = 1; - * @return {string} - */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this - */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.setTypeName = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this - */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.clearTypeName = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.hasTypeName = function() { - return jspb.Message.getField(this, 1) != null; }; /** - * optional string type_version = 2; - * @return {string} + * repeated Execution executions = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.getExecutionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this - */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 2, value); + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this +*/ +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.setExecutionsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this + * @param {!proto.ml_metadata.Execution=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Execution} */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.addExecutions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.hasTypeVersion = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.clearExecutionsList = function() { + return this.setExecutionsList([]); }; /** - * optional TransactionOptions transaction_options = 3; - * @return {?proto.ml_metadata.TransactionOptions} + * optional string next_page_token = 2; + * @return {string} */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.getTransactionOptions = function() { - return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this -*/ -proto.ml_metadata.GetArtifactTypeRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this + */ +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactTypeRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.clearTransactionOptions = function() { - return this.setTransactionOptions(undefined); +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 2, undefined); }; @@ -10647,8 +16956,8 @@ proto.ml_metadata.GetArtifactTypeRequest.prototype.clearTransactionOptions = fun * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactTypeRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetExecutionsByTypeResponse.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -10668,8 +16977,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionByTypeAndNameRequest.toObject(opt_includeInstance, this); }; @@ -10678,13 +16987,16 @@ proto.ml_metadata.GetArtifactTypeResponse.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifactType: (f = msg.getArtifactType()) && ml_metadata_proto_metadata_store_pb.ArtifactType.toObject(includeInstance, f) + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + executionName: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -10698,23 +17010,23 @@ proto.ml_metadata.GetArtifactTypeResponse.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactTypeResponse} + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} */ -proto.ml_metadata.GetArtifactTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactTypeResponse; - return proto.ml_metadata.GetArtifactTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionByTypeAndNameRequest; + return proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactTypeResponse} + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} */ -proto.ml_metadata.GetArtifactTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -10722,9 +17034,21 @@ proto.ml_metadata.GetArtifactTypeResponse.deserializeBinaryFromReader = function var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); - msg.setArtifactType(value); + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); + break; + case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setExecutionName(value); + break; + case 4: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -10739,9 +17063,9 @@ proto.ml_metadata.GetArtifactTypeResponse.deserializeBinaryFromReader = function * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionByTypeAndNameRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -10749,48 +17073,68 @@ proto.ml_metadata.GetArtifactTypeResponse.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactTypeResponse} message + * @param {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactType(); + f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeMessage( + writer.writeString( 1, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 4, f, - ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * optional ArtifactType artifact_type = 1; - * @return {?proto.ml_metadata.ArtifactType} + * optional string type_name = 1; + * @return {string} */ -proto.ml_metadata.GetArtifactTypeResponse.prototype.getArtifactType = function() { - return /** @type{?proto.ml_metadata.ArtifactType} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {?proto.ml_metadata.ArtifactType|undefined} value - * @return {!proto.ml_metadata.GetArtifactTypeResponse} returns this -*/ -proto.ml_metadata.GetArtifactTypeResponse.prototype.setArtifactType = function(value) { - return jspb.Message.setWrapperField(this, 1, value); + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this + */ +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactTypeResponse} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetArtifactTypeResponse.prototype.clearArtifactType = function() { - return this.setArtifactType(undefined); +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -10798,149 +17142,107 @@ proto.ml_metadata.GetArtifactTypeResponse.prototype.clearArtifactType = function * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactTypeResponse.prototype.hasArtifactType = function() { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasTypeName = function() { return jspb.Message.getField(this, 1) != null; }; +/** + * optional string type_version = 3; + * @return {string} + */ +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +}; - -if (jspb.Message.GENERATE_TO_OBJECT) { /** - * Creates an object representation of this proto. - * Field names that are reserved in JavaScript and will be renamed to pb_name. - * Optional fields that are not set will be set to undefined. - * To access a reserved field use, foo.pb_, eg, foo.pb_default. - * For the list of reserved names please see: - * net/proto2/compiler/js/internal/generator.cc#kKeyword. - * @param {boolean=} opt_includeInstance Deprecated. whether to include the - * JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @return {!Object} + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetArtifactTypesRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactTypesRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Static version of the {@see toObject} method. - * @param {boolean|undefined} includeInstance Deprecated. Whether to include - * the JSPB instance for transitional soy proto support: - * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactTypesRequest} msg The msg instance to transform. - * @return {!Object} - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetArtifactTypesRequest.toObject = function(includeInstance, msg) { - var f, obj = { - transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) - }; - - if (includeInstance) { - obj.$jspbMessageInstance = msg; - } - return obj; +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 3, undefined); }; -} /** - * Deserializes binary data (in protobuf wire format). - * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactTypesRequest} + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetArtifactTypesRequest.deserializeBinary = function(bytes) { - var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactTypesRequest; - return proto.ml_metadata.GetArtifactTypesRequest.deserializeBinaryFromReader(msg, reader); +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 3) != null; }; /** - * Deserializes binary data (in protobuf wire format) from the - * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactTypesRequest} msg The message object to deserialize into. - * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactTypesRequest} + * optional string execution_name = 2; + * @return {string} */ -proto.ml_metadata.GetArtifactTypesRequest.deserializeBinaryFromReader = function(msg, reader) { - while (reader.nextField()) { - if (reader.isEndGroup()) { - break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); - msg.setTransactionOptions(value); - break; - default: - reader.skipField(); - break; - } - } - return msg; +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getExecutionName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetArtifactTypesRequest.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactTypesRequest.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setExecutionName = function(value) { + return jspb.Message.setField(this, 2, value); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactTypesRequest} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetArtifactTypesRequest.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getTransactionOptions(); - if (f != null) { - writer.writeMessage( - 1, - f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter - ); - } +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearExecutionName = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasExecutionName = function() { + return jspb.Message.getField(this, 2) != null; }; /** - * optional TransactionOptions transaction_options = 1; + * optional TransactionOptions transaction_options = 4; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactTypesRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 1)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactTypesRequest} returns this + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetArtifactTypesRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactTypesRequest} returns this + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetArtifactTypesRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -10949,19 +17251,12 @@ proto.ml_metadata.GetArtifactTypesRequest.prototype.clearTransactionOptions = fu * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactTypesRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 4) != null; }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetArtifactTypesResponse.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -10977,8 +17272,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactTypesResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactTypesResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionByTypeAndNameResponse.toObject(opt_includeInstance, this); }; @@ -10987,14 +17282,13 @@ proto.ml_metadata.GetArtifactTypesResponse.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactTypesResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypesResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionByTypeAndNameResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), - ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance) + execution: (f = msg.getExecution()) && ml_metadata_proto_metadata_store_pb.Execution.toObject(includeInstance, f) }; if (includeInstance) { @@ -11008,23 +17302,23 @@ proto.ml_metadata.GetArtifactTypesResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactTypesResponse} + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} */ -proto.ml_metadata.GetArtifactTypesResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactTypesResponse; - return proto.ml_metadata.GetArtifactTypesResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionByTypeAndNameResponse; + return proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactTypesResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactTypesResponse} + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} */ -proto.ml_metadata.GetArtifactTypesResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11032,9 +17326,9 @@ proto.ml_metadata.GetArtifactTypesResponse.deserializeBinaryFromReader = functio var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); - msg.addArtifactTypes(value); + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.setExecution(value); break; default: reader.skipField(); @@ -11049,9 +17343,9 @@ proto.ml_metadata.GetArtifactTypesResponse.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactTypesResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactTypesResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionByTypeAndNameResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11059,62 +17353,68 @@ proto.ml_metadata.GetArtifactTypesResponse.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactTypesResponse} message + * @param {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypesResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionByTypeAndNameResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactTypesList(); - if (f.length > 0) { - writer.writeRepeatedMessage( + f = message.getExecution(); + if (f != null) { + writer.writeMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter ); } }; /** - * repeated ArtifactType artifact_types = 1; - * @return {!Array} + * optional Execution execution = 1; + * @return {?proto.ml_metadata.Execution} */ -proto.ml_metadata.GetArtifactTypesResponse.prototype.getArtifactTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); +proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.getExecution = function() { + return /** @type{?proto.ml_metadata.Execution} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactTypesResponse} returns this + * @param {?proto.ml_metadata.Execution|undefined} value + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} returns this */ -proto.ml_metadata.GetArtifactTypesResponse.prototype.setArtifactTypesList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); +proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.setExecution = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ArtifactType=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.ArtifactType} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} returns this */ -proto.ml_metadata.GetArtifactTypesResponse.prototype.addArtifactTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactType, opt_index); +proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.clearExecution = function() { + return this.setExecution(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactTypesResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetArtifactTypesResponse.prototype.clearArtifactTypesList = function() { - return this.setArtifactTypesList([]); +proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.hasExecution = function() { + return jspb.Message.getField(this, 1) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetExecutionsByIDRequest.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -11130,8 +17430,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionTypesRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionTypesRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByIDRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByIDRequest.toObject(opt_includeInstance, this); }; @@ -11140,12 +17440,13 @@ proto.ml_metadata.GetExecutionTypesRequest.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionTypesRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByIDRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypesRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByIDRequest.toObject = function(includeInstance, msg) { var f, obj = { + executionIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -11160,23 +17461,23 @@ proto.ml_metadata.GetExecutionTypesRequest.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionTypesRequest} + * @return {!proto.ml_metadata.GetExecutionsByIDRequest} */ -proto.ml_metadata.GetExecutionTypesRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionTypesRequest; - return proto.ml_metadata.GetExecutionTypesRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByIDRequest; + return proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionTypesRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByIDRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionTypesRequest} + * @return {!proto.ml_metadata.GetExecutionsByIDRequest} */ -proto.ml_metadata.GetExecutionTypesRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11184,6 +17485,12 @@ proto.ml_metadata.GetExecutionTypesRequest.deserializeBinaryFromReader = functio var field = reader.getFieldNumber(); switch (field) { case 1: + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addExecutionIds(values[i]); + } + break; + case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -11201,9 +17508,9 @@ proto.ml_metadata.GetExecutionTypesRequest.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionTypesRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByIDRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionTypesRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByIDRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11211,16 +17518,23 @@ proto.ml_metadata.GetExecutionTypesRequest.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionTypesRequest} message + * @param {!proto.ml_metadata.GetExecutionsByIDRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypesRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByIDRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; + f = message.getExecutionIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 1, + f + ); + } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 1, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -11229,29 +17543,66 @@ proto.ml_metadata.GetExecutionTypesRequest.serializeBinaryToWriter = function(me /** - * optional TransactionOptions transaction_options = 1; + * repeated int64 execution_ids = 1; + * @return {!Array} + */ +proto.ml_metadata.GetExecutionsByIDRequest.prototype.getExecutionIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + */ +proto.ml_metadata.GetExecutionsByIDRequest.prototype.setExecutionIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); +}; + + +/** + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + */ +proto.ml_metadata.GetExecutionsByIDRequest.prototype.addExecutionIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + */ +proto.ml_metadata.GetExecutionsByIDRequest.prototype.clearExecutionIdsList = function() { + return this.setExecutionIdsList([]); +}; + + +/** + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionTypesRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetExecutionsByIDRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 1)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionTypesRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this */ -proto.ml_metadata.GetExecutionTypesRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.GetExecutionsByIDRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionTypesRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this */ -proto.ml_metadata.GetExecutionTypesRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetExecutionsByIDRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -11260,8 +17611,8 @@ proto.ml_metadata.GetExecutionTypesRequest.prototype.clearTransactionOptions = f * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionTypesRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetExecutionsByIDRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -11271,7 +17622,7 @@ proto.ml_metadata.GetExecutionTypesRequest.prototype.hasTransactionOptions = fun * @private {!Array} * @const */ -proto.ml_metadata.GetExecutionTypesResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetExecutionsByIDResponse.repeatedFields_ = [1]; @@ -11288,8 +17639,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionTypesResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionTypesResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByIDResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByIDResponse.toObject(opt_includeInstance, this); }; @@ -11298,14 +17649,14 @@ proto.ml_metadata.GetExecutionTypesResponse.prototype.toObject = function(opt_in * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionTypesResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByIDResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypesResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByIDResponse.toObject = function(includeInstance, msg) { var f, obj = { - executionTypesList: jspb.Message.toObjectList(msg.getExecutionTypesList(), - ml_metadata_proto_metadata_store_pb.ExecutionType.toObject, includeInstance) + executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), + ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance) }; if (includeInstance) { @@ -11319,33 +17670,33 @@ proto.ml_metadata.GetExecutionTypesResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionTypesResponse} + * @return {!proto.ml_metadata.GetExecutionsByIDResponse} */ -proto.ml_metadata.GetExecutionTypesResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionTypesResponse; - return proto.ml_metadata.GetExecutionTypesResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByIDResponse; + return proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionTypesResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByIDResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionTypesResponse} + * @return {!proto.ml_metadata.GetExecutionsByIDResponse} */ -proto.ml_metadata.GetExecutionTypesResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); - msg.addExecutionTypes(value); + switch (field) { + case 1: + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.addExecutions(value); break; default: reader.skipField(); @@ -11360,9 +17711,9 @@ proto.ml_metadata.GetExecutionTypesResponse.deserializeBinaryFromReader = functi * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionTypesResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByIDResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionTypesResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByIDResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11370,58 +17721,58 @@ proto.ml_metadata.GetExecutionTypesResponse.prototype.serializeBinary = function /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionTypesResponse} message + * @param {!proto.ml_metadata.GetExecutionsByIDResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypesResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByIDResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionTypesList(); + f = message.getExecutionsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter ); } }; /** - * repeated ExecutionType execution_types = 1; - * @return {!Array} + * repeated Execution executions = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionTypesResponse.prototype.getExecutionTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); +proto.ml_metadata.GetExecutionsByIDResponse.prototype.getExecutionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionTypesResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionsByIDResponse} returns this */ -proto.ml_metadata.GetExecutionTypesResponse.prototype.setExecutionTypesList = function(value) { +proto.ml_metadata.GetExecutionsByIDResponse.prototype.setExecutionsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ExecutionType=} opt_value + * @param {!proto.ml_metadata.Execution=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.ExecutionType} + * @return {!proto.ml_metadata.Execution} */ -proto.ml_metadata.GetExecutionTypesResponse.prototype.addExecutionTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ExecutionType, opt_index); +proto.ml_metadata.GetExecutionsByIDResponse.prototype.addExecutions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionTypesResponse} returns this + * @return {!proto.ml_metadata.GetExecutionsByIDResponse} returns this */ -proto.ml_metadata.GetExecutionTypesResponse.prototype.clearExecutionTypesList = function() { - return this.setExecutionTypesList([]); +proto.ml_metadata.GetExecutionsByIDResponse.prototype.clearExecutionsList = function() { + return this.setExecutionsList([]); }; @@ -11441,8 +17792,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextTypesRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextTypesRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypeRequest.toObject(opt_includeInstance, this); }; @@ -11451,12 +17802,14 @@ proto.ml_metadata.GetContextTypesRequest.prototype.toObject = function(opt_inclu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextTypesRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypeRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypeRequest.toObject = function(includeInstance, msg) { var f, obj = { + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -11471,23 +17824,23 @@ proto.ml_metadata.GetContextTypesRequest.toObject = function(includeInstance, ms /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextTypesRequest} + * @return {!proto.ml_metadata.GetExecutionTypeRequest} */ -proto.ml_metadata.GetContextTypesRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypeRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextTypesRequest; - return proto.ml_metadata.GetContextTypesRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypeRequest; + return proto.ml_metadata.GetExecutionTypeRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextTypesRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypeRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextTypesRequest} + * @return {!proto.ml_metadata.GetExecutionTypeRequest} */ -proto.ml_metadata.GetContextTypesRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypeRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11495,6 +17848,14 @@ proto.ml_metadata.GetContextTypesRequest.deserializeBinaryFromReader = function( var field = reader.getFieldNumber(); switch (field) { case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); + break; + case 3: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -11512,9 +17873,9 @@ proto.ml_metadata.GetContextTypesRequest.deserializeBinaryFromReader = function( * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextTypesRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypeRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextTypesRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypeRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11522,16 +17883,30 @@ proto.ml_metadata.GetContextTypesRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextTypesRequest} message + * @param {!proto.ml_metadata.GetExecutionTypeRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypeRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( + 1, + f + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 1, + 3, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -11540,29 +17915,101 @@ proto.ml_metadata.GetContextTypesRequest.serializeBinaryToWriter = function(mess /** - * optional TransactionOptions transaction_options = 1; + * optional string type_name = 1; + * @return {string} + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.hasTypeName = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional string type_version = 2; + * @return {string} + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetExecutionTypeRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 3; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextTypesRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetExecutionTypeRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 1)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextTypesRequest} returns this + * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this */ -proto.ml_metadata.GetContextTypesRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.GetExecutionTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextTypesRequest} returns this + * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this */ -proto.ml_metadata.GetContextTypesRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetExecutionTypeRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -11571,19 +18018,12 @@ proto.ml_metadata.GetContextTypesRequest.prototype.clearTransactionOptions = fun * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextTypesRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetExecutionTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetContextTypesResponse.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -11599,8 +18039,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextTypesResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextTypesResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypeResponse.toObject(opt_includeInstance, this); }; @@ -11609,14 +18049,13 @@ proto.ml_metadata.GetContextTypesResponse.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextTypesResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypeResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypeResponse.toObject = function(includeInstance, msg) { var f, obj = { - contextTypesList: jspb.Message.toObjectList(msg.getContextTypesList(), - ml_metadata_proto_metadata_store_pb.ContextType.toObject, includeInstance) + executionType: (f = msg.getExecutionType()) && ml_metadata_proto_metadata_store_pb.ExecutionType.toObject(includeInstance, f) }; if (includeInstance) { @@ -11630,23 +18069,23 @@ proto.ml_metadata.GetContextTypesResponse.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextTypesResponse} + * @return {!proto.ml_metadata.GetExecutionTypeResponse} */ -proto.ml_metadata.GetContextTypesResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypeResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextTypesResponse; - return proto.ml_metadata.GetContextTypesResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypeResponse; + return proto.ml_metadata.GetExecutionTypeResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextTypesResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypeResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextTypesResponse} + * @return {!proto.ml_metadata.GetExecutionTypeResponse} */ -proto.ml_metadata.GetContextTypesResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypeResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11654,9 +18093,9 @@ proto.ml_metadata.GetContextTypesResponse.deserializeBinaryFromReader = function var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ContextType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); - msg.addContextTypes(value); + var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); + msg.setExecutionType(value); break; default: reader.skipField(); @@ -11671,9 +18110,9 @@ proto.ml_metadata.GetContextTypesResponse.deserializeBinaryFromReader = function * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextTypesResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypeResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextTypesResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypeResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11681,62 +18120,68 @@ proto.ml_metadata.GetContextTypesResponse.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextTypesResponse} message + * @param {!proto.ml_metadata.GetExecutionTypeResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypeResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextTypesList(); - if (f.length > 0) { - writer.writeRepeatedMessage( + f = message.getExecutionType(); + if (f != null) { + writer.writeMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter ); } }; /** - * repeated ContextType context_types = 1; - * @return {!Array} - */ -proto.ml_metadata.GetContextTypesResponse.prototype.getContextTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); + * optional ExecutionType execution_type = 1; + * @return {?proto.ml_metadata.ExecutionType} + */ +proto.ml_metadata.GetExecutionTypeResponse.prototype.getExecutionType = function() { + return /** @type{?proto.ml_metadata.ExecutionType} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetContextTypesResponse} returns this + * @param {?proto.ml_metadata.ExecutionType|undefined} value + * @return {!proto.ml_metadata.GetExecutionTypeResponse} returns this */ -proto.ml_metadata.GetContextTypesResponse.prototype.setContextTypesList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); +proto.ml_metadata.GetExecutionTypeResponse.prototype.setExecutionType = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ContextType=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.ContextType} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionTypeResponse} returns this */ -proto.ml_metadata.GetContextTypesResponse.prototype.addContextTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ContextType, opt_index); +proto.ml_metadata.GetExecutionTypeResponse.prototype.clearExecutionType = function() { + return this.setExecutionType(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextTypesResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetContextTypesResponse.prototype.clearContextTypesList = function() { - return this.setContextTypesList([]); +proto.ml_metadata.GetExecutionTypeResponse.prototype.hasExecutionType = function() { + return jspb.Message.getField(this, 1) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetEventsByExecutionIDsRequest.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -11752,8 +18197,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsByTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetEventsByExecutionIDsRequest.toObject(opt_includeInstance, this); }; @@ -11762,15 +18207,13 @@ proto.ml_metadata.GetExecutionsByTypeRequest.prototype.toObject = function(opt_i * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsByTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetEventsByExecutionIDsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetEventsByExecutionIDsRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + executionIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -11785,23 +18228,23 @@ proto.ml_metadata.GetExecutionsByTypeRequest.toObject = function(includeInstance /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} + * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} */ -proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsByTypeRequest; - return proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetEventsByExecutionIDsRequest; + return proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsByTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetEventsByExecutionIDsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} + * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} */ -proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -11809,19 +18252,12 @@ proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinaryFromReader = funct var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addExecutionIds(values[i]); + } break; case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); - break; - case 3: - var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); - msg.setOptions(value); - break; - case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -11839,9 +18275,9 @@ proto.ml_metadata.GetExecutionsByTypeRequest.deserializeBinaryFromReader = funct * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsByTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetEventsByExecutionIDsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -11849,38 +18285,23 @@ proto.ml_metadata.GetExecutionsByTypeRequest.prototype.serializeBinary = functio /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsByTypeRequest} message + * @param {!proto.ml_metadata.GetEventsByExecutionIDsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetEventsByExecutionIDsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( + f = message.getExecutionIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( 1, f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } - f = message.getOptions(); - if (f != null) { - writer.writeMessage( - 3, - f, - ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter - ); - } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 4, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -11889,138 +18310,66 @@ proto.ml_metadata.GetExecutionsByTypeRequest.serializeBinaryToWriter = function( /** - * optional string type_name = 1; - * @return {string} - */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this - */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setTypeName = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this - */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearTypeName = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasTypeName = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * optional string type_version = 2; - * @return {string} - */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this - */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 2, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this - */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 2, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} + * repeated int64 execution_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasTypeVersion = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.getExecutionIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * optional ListOperationOptions options = 3; - * @return {?proto.ml_metadata.ListOperationOptions} + * @param {!Array} value + * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 3)); -}; - - -/** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this -*/ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.setExecutionIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.addExecutionIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.clearExecutionIdsList = function() { + return this.setExecutionIdsList([]); }; /** - * optional TransactionOptions transaction_options = 4; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this + * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByTypeRequest} returns this + * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -12029,8 +18378,8 @@ proto.ml_metadata.GetExecutionsByTypeRequest.prototype.clearTransactionOptions = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -12040,7 +18389,7 @@ proto.ml_metadata.GetExecutionsByTypeRequest.prototype.hasTransactionOptions = f * @private {!Array} * @const */ -proto.ml_metadata.GetExecutionsByTypeResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetEventsByExecutionIDsResponse.repeatedFields_ = [1]; @@ -12057,8 +18406,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsByTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetEventsByExecutionIDsResponse.toObject(opt_includeInstance, this); }; @@ -12067,15 +18416,14 @@ proto.ml_metadata.GetExecutionsByTypeResponse.prototype.toObject = function(opt_ * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsByTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetEventsByExecutionIDsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetEventsByExecutionIDsResponse.toObject = function(includeInstance, msg) { var f, obj = { - executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), - ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f + eventsList: jspb.Message.toObjectList(msg.getEventsList(), + ml_metadata_proto_metadata_store_pb.Event.toObject, includeInstance) }; if (includeInstance) { @@ -12089,23 +18437,23 @@ proto.ml_metadata.GetExecutionsByTypeResponse.toObject = function(includeInstanc /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} + * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} */ -proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsByTypeResponse; - return proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetEventsByExecutionIDsResponse; + return proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsByTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetEventsByExecutionIDsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} + * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} */ -proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -12113,13 +18461,9 @@ proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinaryFromReader = func var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Execution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); - msg.addExecutions(value); - break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); + var value = new ml_metadata_proto_metadata_store_pb.Event; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); + msg.addEvents(value); break; default: reader.skipField(); @@ -12134,9 +18478,9 @@ proto.ml_metadata.GetExecutionsByTypeResponse.deserializeBinaryFromReader = func * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsByTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetEventsByExecutionIDsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -12144,104 +18488,68 @@ proto.ml_metadata.GetExecutionsByTypeResponse.prototype.serializeBinary = functi /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsByTypeResponse} message + * @param {!proto.ml_metadata.GetEventsByExecutionIDsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetEventsByExecutionIDsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionsList(); + f = message.getEventsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f + ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter ); } }; /** - * repeated Execution executions = 1; - * @return {!Array} + * repeated Event events = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.getExecutionsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.getEventsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} returns this */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.setExecutionsList = function(value) { +proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.setEventsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Execution=} opt_value + * @param {!proto.ml_metadata.Event=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Execution} + * @return {!proto.ml_metadata.Event} */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.addExecutions = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); +proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.addEvents = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Event, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this - */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.clearExecutionsList = function() { - return this.setExecutionsList([]); -}; - - -/** - * optional string next_page_token = 2; - * @return {string} - */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this + * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} returns this */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.clearEventsList = function() { + return this.setEventsList([]); }; -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByTypeResponse} returns this - */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 2, undefined); -}; - /** - * Returns whether this field is set. - * @return {boolean} + * List of repeated fields within this message type. + * @private {!Array} + * @const */ -proto.ml_metadata.GetExecutionsByTypeResponse.prototype.hasNextPageToken = function() { - return jspb.Message.getField(this, 2) != null; -}; - - +proto.ml_metadata.GetEventsByArtifactIDsRequest.repeatedFields_ = [1]; @@ -12258,8 +18566,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionByTypeAndNameRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetEventsByArtifactIDsRequest.toObject(opt_includeInstance, this); }; @@ -12268,15 +18576,13 @@ proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.toObject = function * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetEventsByArtifactIDsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetEventsByArtifactIDsRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, - executionName: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -12291,23 +18597,23 @@ proto.ml_metadata.GetExecutionByTypeAndNameRequest.toObject = function(includeIn /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} + * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionByTypeAndNameRequest; - return proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetEventsByArtifactIDsRequest; + return proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetEventsByArtifactIDsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} + * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -12315,18 +18621,12 @@ proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinaryFromReader = var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); - break; - case 3: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addArtifactIds(values[i]); + } break; case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setExecutionName(value); - break; - case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -12344,9 +18644,9 @@ proto.ml_metadata.GetExecutionByTypeAndNameRequest.deserializeBinaryFromReader = * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionByTypeAndNameRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetEventsByArtifactIDsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -12354,176 +18654,91 @@ proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.serializeBinary = f /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} message + * @param {!proto.ml_metadata.GetEventsByArtifactIDsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetEventsByArtifactIDsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( - 1, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeString( - 3, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } - f = message.getTransactionOptions(); - if (f != null) { - writer.writeMessage( - 4, - f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter - ); - } -}; - - -/** - * optional string type_name = 1; - * @return {string} - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setTypeName = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearTypeName = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasTypeName = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * optional string type_version = 3; - * @return {string} - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 3, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 3, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasTypeVersion = function() { - return jspb.Message.getField(this, 3) != null; + f = message.getArtifactIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( + 1, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } }; /** - * optional string execution_name = 2; - * @return {string} + * repeated int64 artifact_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getExecutionName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.getArtifactIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setExecutionName = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.setArtifactIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearExecutionName = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.addArtifactIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasExecutionName = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.clearArtifactIdsList = function() { + return this.setArtifactIdsList([]); }; /** - * optional TransactionOptions transaction_options = 4; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this + * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameRequest} returns this + * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -12532,12 +18747,19 @@ proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.clearTransactionOpt * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionByTypeAndNameRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetEventsByArtifactIDsResponse.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -12553,8 +18775,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionByTypeAndNameResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetEventsByArtifactIDsResponse.toObject(opt_includeInstance, this); }; @@ -12563,13 +18785,14 @@ proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.toObject = functio * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetEventsByArtifactIDsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetEventsByArtifactIDsResponse.toObject = function(includeInstance, msg) { var f, obj = { - execution: (f = msg.getExecution()) && ml_metadata_proto_metadata_store_pb.Execution.toObject(includeInstance, f) + eventsList: jspb.Message.toObjectList(msg.getEventsList(), + ml_metadata_proto_metadata_store_pb.Event.toObject, includeInstance) }; if (includeInstance) { @@ -12583,23 +18806,23 @@ proto.ml_metadata.GetExecutionByTypeAndNameResponse.toObject = function(includeI /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} + * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionByTypeAndNameResponse; - return proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetEventsByArtifactIDsResponse; + return proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetEventsByArtifactIDsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} + * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -12607,9 +18830,9 @@ proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinaryFromReader var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Execution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); - msg.setExecution(value); + var value = new ml_metadata_proto_metadata_store_pb.Event; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); + msg.addEvents(value); break; default: reader.skipField(); @@ -12624,9 +18847,9 @@ proto.ml_metadata.GetExecutionByTypeAndNameResponse.deserializeBinaryFromReader * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionByTypeAndNameResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetEventsByArtifactIDsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -12634,57 +18857,58 @@ proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.serializeBinary = /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} message + * @param {!proto.ml_metadata.GetEventsByArtifactIDsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetEventsByArtifactIDsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecution(); - if (f != null) { - writer.writeMessage( + f = message.getEventsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter ); } }; /** - * optional Execution execution = 1; - * @return {?proto.ml_metadata.Execution} + * repeated Event events = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.getExecution = function() { - return /** @type{?proto.ml_metadata.Execution} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.getEventsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 1)); }; /** - * @param {?proto.ml_metadata.Execution|undefined} value - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} returns this */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.setExecution = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.setEventsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionByTypeAndNameResponse} returns this + * @param {!proto.ml_metadata.Event=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Event} */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.clearExecution = function() { - return this.setExecution(undefined); +proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.addEvents = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Event, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} returns this */ -proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.hasExecution = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.clearEventsList = function() { + return this.setEventsList([]); }; @@ -12694,7 +18918,7 @@ proto.ml_metadata.GetExecutionByTypeAndNameResponse.prototype.hasExecution = fun * @private {!Array} * @const */ -proto.ml_metadata.GetExecutionsByIDRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactTypesByIDRequest.repeatedFields_ = [1]; @@ -12711,8 +18935,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsByIDRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypesByIDRequest.toObject(opt_includeInstance, this); }; @@ -12721,13 +18945,13 @@ proto.ml_metadata.GetExecutionsByIDRequest.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsByIDRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactTypesByIDRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByIDRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactTypesByIDRequest.toObject = function(includeInstance, msg) { var f, obj = { - executionIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + typeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -12742,23 +18966,23 @@ proto.ml_metadata.GetExecutionsByIDRequest.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsByIDRequest} + * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} */ -proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsByIDRequest; - return proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactTypesByIDRequest; + return proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsByIDRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactTypesByIDRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsByIDRequest} + * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} */ -proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -12768,7 +18992,7 @@ proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinaryFromReader = functio case 1: var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); for (var i = 0; i < values.length; i++) { - msg.addExecutionIds(values[i]); + msg.addTypeIds(values[i]); } break; case 2: @@ -12789,9 +19013,9 @@ proto.ml_metadata.GetExecutionsByIDRequest.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsByIDRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactTypesByIDRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -12799,13 +19023,13 @@ proto.ml_metadata.GetExecutionsByIDRequest.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsByIDRequest} message + * @param {!proto.ml_metadata.GetArtifactTypesByIDRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByIDRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactTypesByIDRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionIdsList(); + f = message.getTypeIdsList(); if (f.length > 0) { writer.writeRepeatedInt64( 1, @@ -12824,19 +19048,19 @@ proto.ml_metadata.GetExecutionsByIDRequest.serializeBinaryToWriter = function(me /** - * repeated int64 execution_ids = 1; + * repeated int64 type_ids = 1; * @return {!Array} */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.getExecutionIdsList = function() { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.getTypeIdsList = function() { return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.setExecutionIdsList = function(value) { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.setTypeIdsList = function(value) { return jspb.Message.setField(this, 1, value || []); }; @@ -12844,19 +19068,19 @@ proto.ml_metadata.GetExecutionsByIDRequest.prototype.setExecutionIdsList = funct /** * @param {number} value * @param {number=} opt_index - * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.addExecutionIds = function(value, opt_index) { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.addTypeIds = function(value, opt_index) { return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.clearExecutionIdsList = function() { - return this.setExecutionIdsList([]); +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.clearTypeIdsList = function() { + return this.setTypeIdsList([]); }; @@ -12864,7 +19088,7 @@ proto.ml_metadata.GetExecutionsByIDRequest.prototype.clearExecutionIdsList = fun * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -12872,18 +19096,18 @@ proto.ml_metadata.GetExecutionsByIDRequest.prototype.getTransactionOptions = fun /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByIDRequest} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -12892,7 +19116,7 @@ proto.ml_metadata.GetExecutionsByIDRequest.prototype.clearTransactionOptions = f * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionsByIDRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -12903,7 +19127,7 @@ proto.ml_metadata.GetExecutionsByIDRequest.prototype.hasTransactionOptions = fun * @private {!Array} * @const */ -proto.ml_metadata.GetExecutionsByIDResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactTypesByIDResponse.repeatedFields_ = [1]; @@ -12920,8 +19144,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionsByIDResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsByIDResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactTypesByIDResponse.toObject(opt_includeInstance, this); }; @@ -12930,14 +19154,14 @@ proto.ml_metadata.GetExecutionsByIDResponse.prototype.toObject = function(opt_in * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsByIDResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactTypesByIDResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByIDResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactTypesByIDResponse.toObject = function(includeInstance, msg) { var f, obj = { - executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), - ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance) + artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), + ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance) }; if (includeInstance) { @@ -12951,23 +19175,23 @@ proto.ml_metadata.GetExecutionsByIDResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsByIDResponse} + * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} */ -proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsByIDResponse; - return proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactTypesByIDResponse; + return proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsByIDResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactTypesByIDResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsByIDResponse} + * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} */ -proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -12975,9 +19199,9 @@ proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinaryFromReader = functi var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Execution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); - msg.addExecutions(value); + var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); + msg.addArtifactTypes(value); break; default: reader.skipField(); @@ -12992,9 +19216,9 @@ proto.ml_metadata.GetExecutionsByIDResponse.deserializeBinaryFromReader = functi * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionsByIDResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsByIDResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactTypesByIDResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -13002,62 +19226,69 @@ proto.ml_metadata.GetExecutionsByIDResponse.prototype.serializeBinary = function /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsByIDResponse} message + * @param {!proto.ml_metadata.GetArtifactTypesByIDResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByIDResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactTypesByIDResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionsList(); + f = message.getArtifactTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter ); } }; /** - * repeated Execution executions = 1; - * @return {!Array} + * repeated ArtifactType artifact_types = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionsByIDResponse.prototype.getExecutionsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); +proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.getArtifactTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionsByIDResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} returns this */ -proto.ml_metadata.GetExecutionsByIDResponse.prototype.setExecutionsList = function(value) { +proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.setArtifactTypesList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Execution=} opt_value + * @param {!proto.ml_metadata.ArtifactType=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Execution} + * @return {!proto.ml_metadata.ArtifactType} */ -proto.ml_metadata.GetExecutionsByIDResponse.prototype.addExecutions = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); +proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.addArtifactTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactType, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionsByIDResponse} returns this + * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} returns this */ -proto.ml_metadata.GetExecutionsByIDResponse.prototype.clearExecutionsList = function() { - return this.setExecutionsList([]); +proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.clearArtifactTypesList = function() { + return this.setArtifactTypesList([]); }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetExecutionTypesByIDRequest.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -13073,8 +19304,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypesByIDRequest.toObject(opt_includeInstance, this); }; @@ -13083,14 +19314,13 @@ proto.ml_metadata.GetExecutionTypeRequest.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypesByIDRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypesByIDRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + typeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -13105,23 +19335,23 @@ proto.ml_metadata.GetExecutionTypeRequest.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionTypeRequest} + * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} */ -proto.ml_metadata.GetExecutionTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionTypeRequest; - return proto.ml_metadata.GetExecutionTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypesByIDRequest; + return proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypesByIDRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionTypeRequest} + * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} */ -proto.ml_metadata.GetExecutionTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -13129,14 +19359,12 @@ proto.ml_metadata.GetExecutionTypeRequest.deserializeBinaryFromReader = function var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addTypeIds(values[i]); + } break; case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); - break; - case 3: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -13154,9 +19382,9 @@ proto.ml_metadata.GetExecutionTypeRequest.deserializeBinaryFromReader = function * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypesByIDRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -13164,30 +19392,23 @@ proto.ml_metadata.GetExecutionTypeRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionTypeRequest} message + * @param {!proto.ml_metadata.GetExecutionTypesByIDRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypesByIDRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeString( + f = message.getTypeIdsList(); + if (f.length > 0) { + writer.writeRepeatedInt64( 1, f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 3, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -13196,101 +19417,66 @@ proto.ml_metadata.GetExecutionTypeRequest.serializeBinaryToWriter = function(mes /** - * optional string type_name = 1; - * @return {string} - */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this - */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.setTypeName = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this - */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.clearTypeName = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.hasTypeName = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * optional string type_version = 2; - * @return {string} + * repeated int64 type_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.getTypeIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.setTypeIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.addTypeIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.hasTypeVersion = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.clearTypeIdsList = function() { + return this.setTypeIdsList([]); }; /** - * optional TransactionOptions transaction_options = 3; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionTypeRequest} returns this + * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -13299,12 +19485,19 @@ proto.ml_metadata.GetExecutionTypeRequest.prototype.clearTransactionOptions = fu * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionTypeRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetExecutionTypesByIDResponse.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -13320,8 +19513,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionTypesByIDResponse.toObject(opt_includeInstance, this); }; @@ -13330,13 +19523,14 @@ proto.ml_metadata.GetExecutionTypeResponse.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionTypesByIDResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionTypesByIDResponse.toObject = function(includeInstance, msg) { var f, obj = { - executionType: (f = msg.getExecutionType()) && ml_metadata_proto_metadata_store_pb.ExecutionType.toObject(includeInstance, f) + executionTypesList: jspb.Message.toObjectList(msg.getExecutionTypesList(), + ml_metadata_proto_metadata_store_pb.ExecutionType.toObject, includeInstance) }; if (includeInstance) { @@ -13350,23 +19544,23 @@ proto.ml_metadata.GetExecutionTypeResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionTypeResponse} + * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} */ -proto.ml_metadata.GetExecutionTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionTypeResponse; - return proto.ml_metadata.GetExecutionTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionTypesByIDResponse; + return proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionTypesByIDResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionTypeResponse} + * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} */ -proto.ml_metadata.GetExecutionTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -13376,7 +19570,7 @@ proto.ml_metadata.GetExecutionTypeResponse.deserializeBinaryFromReader = functio case 1: var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); - msg.setExecutionType(value); + msg.addExecutionTypes(value); break; default: reader.skipField(); @@ -13391,9 +19585,9 @@ proto.ml_metadata.GetExecutionTypeResponse.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionTypesByIDResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -13401,15 +19595,15 @@ proto.ml_metadata.GetExecutionTypeResponse.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionTypeResponse} message + * @param {!proto.ml_metadata.GetExecutionTypesByIDResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionTypesByIDResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionType(); - if (f != null) { - writer.writeMessage( + f = message.getExecutionTypesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, f, ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter @@ -13419,50 +19613,44 @@ proto.ml_metadata.GetExecutionTypeResponse.serializeBinaryToWriter = function(me /** - * optional ExecutionType execution_type = 1; - * @return {?proto.ml_metadata.ExecutionType} + * repeated ExecutionType execution_types = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionTypeResponse.prototype.getExecutionType = function() { - return /** @type{?proto.ml_metadata.ExecutionType} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); +proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.getExecutionTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); }; /** - * @param {?proto.ml_metadata.ExecutionType|undefined} value - * @return {!proto.ml_metadata.GetExecutionTypeResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} returns this */ -proto.ml_metadata.GetExecutionTypeResponse.prototype.setExecutionType = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.setExecutionTypesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionTypeResponse} returns this + * @param {!proto.ml_metadata.ExecutionType=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.ExecutionType} */ -proto.ml_metadata.GetExecutionTypeResponse.prototype.clearExecutionType = function() { - return this.setExecutionType(undefined); +proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.addExecutionTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ExecutionType, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} returns this */ -proto.ml_metadata.GetExecutionTypeResponse.prototype.hasExecutionType = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.clearExecutionTypesList = function() { + return this.setExecutionTypesList([]); }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -13478,8 +19666,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetEventsByExecutionIDsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypeRequest.toObject(opt_includeInstance, this); }; @@ -13488,13 +19676,14 @@ proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.toObject = function(o * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetEventsByExecutionIDsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypeRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypeRequest.toObject = function(includeInstance, msg) { var f, obj = { - executionIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -13509,23 +19698,23 @@ proto.ml_metadata.GetEventsByExecutionIDsRequest.toObject = function(includeInst /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} + * @return {!proto.ml_metadata.GetContextTypeRequest} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypeRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetEventsByExecutionIDsRequest; - return proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypeRequest; + return proto.ml_metadata.GetContextTypeRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetEventsByExecutionIDsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypeRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} + * @return {!proto.ml_metadata.GetContextTypeRequest} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypeRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -13533,12 +19722,14 @@ proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinaryFromReader = f var field = reader.getFieldNumber(); switch (field) { case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addExecutionIds(values[i]); - } + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); break; case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); + break; + case 3: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -13556,9 +19747,9 @@ proto.ml_metadata.GetEventsByExecutionIDsRequest.deserializeBinaryFromReader = f * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypeRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetEventsByExecutionIDsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypeRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -13566,23 +19757,30 @@ proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.serializeBinary = fun /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetEventsByExecutionIDsRequest} message + * @param {!proto.ml_metadata.GetContextTypeRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypeRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( 1, f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f + ); + } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 2, + 3, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -13591,66 +19789,101 @@ proto.ml_metadata.GetEventsByExecutionIDsRequest.serializeBinaryToWriter = funct /** - * repeated int64 execution_ids = 1; - * @return {!Array} + * optional string type_name = 1; + * @return {string} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.getExecutionIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +proto.ml_metadata.GetContextTypeRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetContextTypeRequest} returns this */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.setExecutionIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); +proto.ml_metadata.GetContextTypeRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextTypeRequest} returns this */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.addExecutionIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.GetContextTypeRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.clearExecutionIdsList = function() { - return this.setExecutionIdsList([]); +proto.ml_metadata.GetContextTypeRequest.prototype.hasTypeName = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional TransactionOptions transaction_options = 2; + * optional string type_version = 2; + * @return {string} + */ +proto.ml_metadata.GetContextTypeRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + */ +proto.ml_metadata.GetContextTypeRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + */ +proto.ml_metadata.GetContextTypeRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetContextTypeRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 3; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextTypeRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this + * @return {!proto.ml_metadata.GetContextTypeRequest} returns this */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.ml_metadata.GetContextTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetEventsByExecutionIDsRequest} returns this + * @return {!proto.ml_metadata.GetContextTypeRequest} returns this */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextTypeRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -13659,19 +19892,12 @@ proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.clearTransactionOptio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetEventsByExecutionIDsRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetContextTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -13687,8 +19913,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetEventsByExecutionIDsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypeResponse.toObject(opt_includeInstance, this); }; @@ -13697,14 +19923,13 @@ proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.toObject = function( * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetEventsByExecutionIDsResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypeResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypeResponse.toObject = function(includeInstance, msg) { var f, obj = { - eventsList: jspb.Message.toObjectList(msg.getEventsList(), - ml_metadata_proto_metadata_store_pb.Event.toObject, includeInstance) + contextType: (f = msg.getContextType()) && ml_metadata_proto_metadata_store_pb.ContextType.toObject(includeInstance, f) }; if (includeInstance) { @@ -13718,23 +19943,23 @@ proto.ml_metadata.GetEventsByExecutionIDsResponse.toObject = function(includeIns /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} + * @return {!proto.ml_metadata.GetContextTypeResponse} */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypeResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetEventsByExecutionIDsResponse; - return proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypeResponse; + return proto.ml_metadata.GetContextTypeResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetEventsByExecutionIDsResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypeResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} + * @return {!proto.ml_metadata.GetContextTypeResponse} */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypeResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -13742,9 +19967,9 @@ proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinaryFromReader = var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Event; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); - msg.addEvents(value); + var value = new ml_metadata_proto_metadata_store_pb.ContextType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); + msg.setContextType(value); break; default: reader.skipField(); @@ -13759,9 +19984,9 @@ proto.ml_metadata.GetEventsByExecutionIDsResponse.deserializeBinaryFromReader = * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypeResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetEventsByExecutionIDsResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypeResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -13769,58 +19994,57 @@ proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.serializeBinary = fu /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetEventsByExecutionIDsResponse} message + * @param {!proto.ml_metadata.GetContextTypeResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypeResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getEventsList(); - if (f.length > 0) { - writer.writeRepeatedMessage( + f = message.getContextType(); + if (f != null) { + writer.writeMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter ); } }; /** - * repeated Event events = 1; - * @return {!Array} + * optional ContextType context_type = 1; + * @return {?proto.ml_metadata.ContextType} */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.getEventsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 1)); +proto.ml_metadata.GetContextTypeResponse.prototype.getContextType = function() { + return /** @type{?proto.ml_metadata.ContextType} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} returns this + * @param {?proto.ml_metadata.ContextType|undefined} value + * @return {!proto.ml_metadata.GetContextTypeResponse} returns this */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.setEventsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); +proto.ml_metadata.GetContextTypeResponse.prototype.setContextType = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Event=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.Event} + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetContextTypeResponse} returns this */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.addEvents = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Event, opt_index); +proto.ml_metadata.GetContextTypeResponse.prototype.clearContextType = function() { + return this.setContextType(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetEventsByExecutionIDsResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.clearEventsList = function() { - return this.setEventsList([]); +proto.ml_metadata.GetContextTypeResponse.prototype.hasContextType = function() { + return jspb.Message.getField(this, 1) != null; }; @@ -13830,7 +20054,7 @@ proto.ml_metadata.GetEventsByExecutionIDsResponse.prototype.clearEventsList = fu * @private {!Array} * @const */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetContextTypesByIDRequest.repeatedFields_ = [1]; @@ -13847,8 +20071,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetEventsByArtifactIDsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypesByIDRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypesByIDRequest.toObject(opt_includeInstance, this); }; @@ -13857,13 +20081,13 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.toObject = function(op * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetEventsByArtifactIDsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypesByIDRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypesByIDRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifactIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + typeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -13878,23 +20102,23 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.toObject = function(includeInsta /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} + * @return {!proto.ml_metadata.GetContextTypesByIDRequest} */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetEventsByArtifactIDsRequest; - return proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypesByIDRequest; + return proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetEventsByArtifactIDsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypesByIDRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} + * @return {!proto.ml_metadata.GetContextTypesByIDRequest} */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -13904,7 +20128,7 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinaryFromReader = fu case 1: var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); for (var i = 0; i < values.length; i++) { - msg.addArtifactIds(values[i]); + msg.addTypeIds(values[i]); } break; case 2: @@ -13925,9 +20149,9 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.deserializeBinaryFromReader = fu * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetEventsByArtifactIDsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypesByIDRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -13935,13 +20159,13 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.serializeBinary = func /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetEventsByArtifactIDsRequest} message + * @param {!proto.ml_metadata.GetContextTypesByIDRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypesByIDRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactIdsList(); + f = message.getTypeIdsList(); if (f.length > 0) { writer.writeRepeatedInt64( 1, @@ -13960,19 +20184,19 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.serializeBinaryToWriter = functi /** - * repeated int64 artifact_ids = 1; + * repeated int64 type_ids = 1; * @return {!Array} */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.getArtifactIdsList = function() { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.getTypeIdsList = function() { return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** * @param {!Array} value - * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.setArtifactIdsList = function(value) { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.setTypeIdsList = function(value) { return jspb.Message.setField(this, 1, value || []); }; @@ -13980,19 +20204,19 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.setArtifactIdsList = f /** * @param {number} value * @param {number=} opt_index - * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.addArtifactIds = function(value, opt_index) { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.addTypeIds = function(value, opt_index) { return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.clearArtifactIdsList = function() { - return this.setArtifactIdsList([]); +proto.ml_metadata.GetContextTypesByIDRequest.prototype.clearTypeIdsList = function() { + return this.setTypeIdsList([]); }; @@ -14000,7 +20224,7 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.clearArtifactIdsList = * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -14008,18 +20232,18 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.getTransactionOptions /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetEventsByArtifactIDsRequest} returns this + * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -14028,7 +20252,7 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.clearTransactionOption * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetContextTypesByIDRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -14039,7 +20263,7 @@ proto.ml_metadata.GetEventsByArtifactIDsRequest.prototype.hasTransactionOptions * @private {!Array} * @const */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetContextTypesByIDResponse.repeatedFields_ = [1]; @@ -14056,8 +20280,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetEventsByArtifactIDsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextTypesByIDResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextTypesByIDResponse.toObject(opt_includeInstance, this); }; @@ -14066,14 +20290,14 @@ proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.toObject = function(o * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetEventsByArtifactIDsResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextTypesByIDResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextTypesByIDResponse.toObject = function(includeInstance, msg) { var f, obj = { - eventsList: jspb.Message.toObjectList(msg.getEventsList(), - ml_metadata_proto_metadata_store_pb.Event.toObject, includeInstance) + contextTypesList: jspb.Message.toObjectList(msg.getContextTypesList(), + ml_metadata_proto_metadata_store_pb.ContextType.toObject, includeInstance) }; if (includeInstance) { @@ -14087,23 +20311,23 @@ proto.ml_metadata.GetEventsByArtifactIDsResponse.toObject = function(includeInst /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} + * @return {!proto.ml_metadata.GetContextTypesByIDResponse} */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetEventsByArtifactIDsResponse; - return proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextTypesByIDResponse; + return proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetEventsByArtifactIDsResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextTypesByIDResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} + * @return {!proto.ml_metadata.GetContextTypesByIDResponse} */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -14111,9 +20335,9 @@ proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinaryFromReader = f var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Event; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Event.deserializeBinaryFromReader); - msg.addEvents(value); + var value = new ml_metadata_proto_metadata_store_pb.ContextType; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); + msg.addContextTypes(value); break; default: reader.skipField(); @@ -14128,9 +20352,9 @@ proto.ml_metadata.GetEventsByArtifactIDsResponse.deserializeBinaryFromReader = f * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextTypesByIDResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetEventsByArtifactIDsResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextTypesByIDResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -14138,69 +20362,62 @@ proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.serializeBinary = fun /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetEventsByArtifactIDsResponse} message + * @param {!proto.ml_metadata.GetContextTypesByIDResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextTypesByIDResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getEventsList(); + f = message.getContextTypesList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Event.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter ); } }; /** - * repeated Event events = 1; - * @return {!Array} + * repeated ContextType context_types = 1; + * @return {!Array} */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.getEventsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Event, 1)); +proto.ml_metadata.GetContextTypesByIDResponse.prototype.getContextTypesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextTypesByIDResponse} returns this */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.setEventsList = function(value) { +proto.ml_metadata.GetContextTypesByIDResponse.prototype.setContextTypesList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Event=} opt_value + * @param {!proto.ml_metadata.ContextType=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Event} + * @return {!proto.ml_metadata.ContextType} */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.addEvents = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Event, opt_index); +proto.ml_metadata.GetContextTypesByIDResponse.prototype.addContextTypes = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ContextType, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetEventsByArtifactIDsResponse} returns this + * @return {!proto.ml_metadata.GetContextTypesByIDResponse} returns this */ -proto.ml_metadata.GetEventsByArtifactIDsResponse.prototype.clearEventsList = function() { - return this.setEventsList([]); +proto.ml_metadata.GetContextTypesByIDResponse.prototype.clearContextTypesList = function() { + return this.setContextTypesList([]); }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetArtifactTypesByIDRequest.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -14216,8 +20433,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactTypesByIDRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsRequest.toObject(opt_includeInstance, this); }; @@ -14226,13 +20443,13 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.toObject = function(opt_ * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactTypesByIDRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypesByIDRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -14247,23 +20464,23 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.toObject = function(includeInstanc /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} + * @return {!proto.ml_metadata.GetContextsRequest} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactTypesByIDRequest; - return proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsRequest; + return proto.ml_metadata.GetContextsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactTypesByIDRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} + * @return {!proto.ml_metadata.GetContextsRequest} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -14271,10 +20488,9 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinaryFromReader = func var field = reader.getFieldNumber(); switch (field) { case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addTypeIds(values[i]); - } + var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); + msg.setOptions(value); break; case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; @@ -14294,9 +20510,9 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.deserializeBinaryFromReader = func * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactTypesByIDRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -14304,17 +20520,18 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.serializeBinary = functi /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactTypesByIDRequest} message + * @param {!proto.ml_metadata.GetContextsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypesByIDRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getTypeIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( + f = message.getOptions(); + if (f != null) { + writer.writeMessage( 1, - f + f, + ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter ); } f = message.getTransactionOptions(); @@ -14329,39 +20546,39 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.serializeBinaryToWriter = function /** - * repeated int64 type_ids = 1; - * @return {!Array} + * optional ListOperationOptions options = 1; + * @return {?proto.ml_metadata.ListOperationOptions} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.getTypeIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +proto.ml_metadata.GetContextsRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this - */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.setTypeIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.GetContextsRequest} returns this +*/ +proto.ml_metadata.GetContextsRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetContextsRequest} returns this */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.addTypeIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.GetContextsRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.clearTypeIdsList = function() { - return this.setTypeIdsList([]); +proto.ml_metadata.GetContextsRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 1) != null; }; @@ -14369,7 +20586,7 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.clearTypeIdsList = funct * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -14377,18 +20594,18 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.getTransactionOptions = /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsRequest} returns this */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetContextsRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsRequest} returns this */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -14397,7 +20614,7 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.clearTransactionOptions * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetContextsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -14408,7 +20625,7 @@ proto.ml_metadata.GetArtifactTypesByIDRequest.prototype.hasTransactionOptions = * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactTypesByIDResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetContextsResponse.repeatedFields_ = [1]; @@ -14425,8 +20642,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactTypesByIDResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsResponse.toObject(opt_includeInstance, this); }; @@ -14435,14 +20652,15 @@ proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.toObject = function(opt * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactTypesByIDResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypesByIDResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactTypesList: jspb.Message.toObjectList(msg.getArtifactTypesList(), - ml_metadata_proto_metadata_store_pb.ArtifactType.toObject, includeInstance) + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f }; if (includeInstance) { @@ -14456,23 +20674,23 @@ proto.ml_metadata.GetArtifactTypesByIDResponse.toObject = function(includeInstan /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} + * @return {!proto.ml_metadata.GetContextsResponse} */ -proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactTypesByIDResponse; - return proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsResponse; + return proto.ml_metadata.GetContextsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactTypesByIDResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} + * @return {!proto.ml_metadata.GetContextsResponse} */ -proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -14480,9 +20698,13 @@ proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinaryFromReader = fun var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ArtifactType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ArtifactType.deserializeBinaryFromReader); - msg.addArtifactTypes(value); + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addContexts(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setNextPageToken(value); break; default: reader.skipField(); @@ -14497,9 +20719,9 @@ proto.ml_metadata.GetArtifactTypesByIDResponse.deserializeBinaryFromReader = fun * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactTypesByIDResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -14507,68 +20729,104 @@ proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.serializeBinary = funct /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactTypesByIDResponse} message + * @param {!proto.ml_metadata.GetContextsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactTypesByIDResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactTypesList(); + f = message.getContextsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ArtifactType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f ); } }; /** - * repeated ArtifactType artifact_types = 1; - * @return {!Array} + * repeated Context contexts = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.getArtifactTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ArtifactType, 1)); +proto.ml_metadata.GetContextsResponse.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextsResponse} returns this */ -proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.setArtifactTypesList = function(value) { +proto.ml_metadata.GetContextsResponse.prototype.setContextsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ArtifactType=} opt_value + * @param {!proto.ml_metadata.Context=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.ArtifactType} + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.addArtifactTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ArtifactType, opt_index); +proto.ml_metadata.GetContextsResponse.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactTypesByIDResponse} returns this + * @return {!proto.ml_metadata.GetContextsResponse} returns this */ -proto.ml_metadata.GetArtifactTypesByIDResponse.prototype.clearArtifactTypesList = function() { - return this.setArtifactTypesList([]); +proto.ml_metadata.GetContextsResponse.prototype.clearContextsList = function() { + return this.setContextsList([]); +}; + + +/** + * optional string next_page_token = 2; + * @return {string} + */ +proto.ml_metadata.GetContextsResponse.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; +/** + * @param {string} value + * @return {!proto.ml_metadata.GetContextsResponse} returns this + */ +proto.ml_metadata.GetContextsResponse.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 2, value); +}; + /** - * List of repeated fields within this message type. - * @private {!Array} - * @const + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextsResponse} returns this */ -proto.ml_metadata.GetExecutionTypesByIDRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetContextsResponse.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetContextsResponse.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 2) != null; +}; + + @@ -14585,8 +20843,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionTypesByIDRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByTypeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByTypeRequest.toObject(opt_includeInstance, this); }; @@ -14595,13 +20853,15 @@ proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.toObject = function(opt * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionTypesByIDRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByTypeRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypesByIDRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByTypeRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -14616,23 +20876,23 @@ proto.ml_metadata.GetExecutionTypesByIDRequest.toObject = function(includeInstan /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} + * @return {!proto.ml_metadata.GetContextsByTypeRequest} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByTypeRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionTypesByIDRequest; - return proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByTypeRequest; + return proto.ml_metadata.GetContextsByTypeRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionTypesByIDRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByTypeRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} + * @return {!proto.ml_metadata.GetContextsByTypeRequest} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByTypeRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -14640,12 +20900,19 @@ proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinaryFromReader = fun var field = reader.getFieldNumber(); switch (field) { case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addTypeIds(values[i]); - } + var value = /** @type {string} */ (reader.readString()); + msg.setTypeName(value); break; case 2: + var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); + msg.setOptions(value); + break; + case 3: + var value = /** @type {string} */ (reader.readString()); + msg.setTypeVersion(value); + break; + case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -14660,104 +20927,191 @@ proto.ml_metadata.GetExecutionTypesByIDRequest.deserializeBinaryFromReader = fun /** - * Serializes the message to binary data (in protobuf wire format). - * @return {!Uint8Array} + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.ml_metadata.GetContextsByTypeRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.ml_metadata.GetContextsByTypeRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.ml_metadata.GetContextsByTypeRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.ml_metadata.GetContextsByTypeRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = /** @type {string} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeString( + 1, + f + ); + } + f = message.getOptions(); + if (f != null) { + writer.writeMessage( + 2, + f, + ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 4, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } +}; + + +/** + * optional string type_name = 1; + * @return {string} + */ +proto.ml_metadata.GetContextsByTypeRequest.prototype.getTypeName = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this + */ +proto.ml_metadata.GetContextsByTypeRequest.prototype.setTypeName = function(value) { + return jspb.Message.setField(this, 1, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this + */ +proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTypeName = function() { + return jspb.Message.setField(this, 1, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetContextsByTypeRequest.prototype.hasTypeName = function() { + return jspb.Message.getField(this, 1) != null; +}; + + +/** + * optional ListOperationOptions options = 2; + * @return {?proto.ml_metadata.ListOperationOptions} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.serializeBinary = function() { - var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionTypesByIDRequest.serializeBinaryToWriter(this, writer); - return writer.getResultBuffer(); +proto.ml_metadata.GetContextsByTypeRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 2)); }; /** - * Serializes the given message to binary data (in protobuf wire - * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionTypesByIDRequest} message - * @param {!jspb.BinaryWriter} writer - * @suppress {unusedLocalVariables} f is only used for nested messages + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this +*/ +proto.ml_metadata.GetContextsByTypeRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionTypesByIDRequest.serializeBinaryToWriter = function(message, writer) { - var f = undefined; - f = message.getTypeIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( - 1, - f - ); - } - f = message.getTransactionOptions(); - if (f != null) { - writer.writeMessage( - 2, - f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter - ); - } +proto.ml_metadata.GetContextsByTypeRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); }; /** - * repeated int64 type_ids = 1; - * @return {!Array} + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.getTypeIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +proto.ml_metadata.GetContextsByTypeRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 2) != null; }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this + * optional string type_version = 3; + * @return {string} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.setTypeIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); +proto.ml_metadata.GetContextsByTypeRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this + * @param {string} value + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.addTypeIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.GetContextsByTypeRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 3, value); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.clearTypeIdsList = function() { - return this.setTypeIdsList([]); +proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 3, undefined); }; /** - * optional TransactionOptions transaction_options = 2; + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetContextsByTypeRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 4; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextsByTypeRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.ml_metadata.GetContextsByTypeRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -14766,8 +21120,8 @@ proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.clearTransactionOptions * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetContextsByTypeRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 4) != null; }; @@ -14777,7 +21131,7 @@ proto.ml_metadata.GetExecutionTypesByIDRequest.prototype.hasTransactionOptions = * @private {!Array} * @const */ -proto.ml_metadata.GetExecutionTypesByIDResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetContextsByTypeResponse.repeatedFields_ = [1]; @@ -14794,8 +21148,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionTypesByIDResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByTypeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByTypeResponse.toObject(opt_includeInstance, this); }; @@ -14804,14 +21158,15 @@ proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.toObject = function(op * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionTypesByIDResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByTypeResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypesByIDResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByTypeResponse.toObject = function(includeInstance, msg) { var f, obj = { - executionTypesList: jspb.Message.toObjectList(msg.getExecutionTypesList(), - ml_metadata_proto_metadata_store_pb.ExecutionType.toObject, includeInstance) + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f }; if (includeInstance) { @@ -14825,23 +21180,23 @@ proto.ml_metadata.GetExecutionTypesByIDResponse.toObject = function(includeInsta /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} + * @return {!proto.ml_metadata.GetContextsByTypeResponse} */ -proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByTypeResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionTypesByIDResponse; - return proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByTypeResponse; + return proto.ml_metadata.GetContextsByTypeResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionTypesByIDResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByTypeResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} + * @return {!proto.ml_metadata.GetContextsByTypeResponse} */ -proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByTypeResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -14849,9 +21204,13 @@ proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinaryFromReader = fu var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ExecutionType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ExecutionType.deserializeBinaryFromReader); - msg.addExecutionTypes(value); + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addContexts(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setNextPageToken(value); break; default: reader.skipField(); @@ -14866,9 +21225,9 @@ proto.ml_metadata.GetExecutionTypesByIDResponse.deserializeBinaryFromReader = fu * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByTypeResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionTypesByIDResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByTypeResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -14876,58 +21235,101 @@ proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.serializeBinary = func /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionTypesByIDResponse} message + * @param {!proto.ml_metadata.GetContextsByTypeResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionTypesByIDResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByTypeResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionTypesList(); + f = message.getContextsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ExecutionType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f ); } }; /** - * repeated ExecutionType execution_types = 1; - * @return {!Array} + * repeated Context contexts = 1; + * @return {!Array} */ -proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.getExecutionTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ExecutionType, 1)); +proto.ml_metadata.GetContextsByTypeResponse.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this */ -proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.setExecutionTypesList = function(value) { +proto.ml_metadata.GetContextsByTypeResponse.prototype.setContextsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ExecutionType=} opt_value + * @param {!proto.ml_metadata.Context=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.ExecutionType} + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.addExecutionTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ExecutionType, opt_index); +proto.ml_metadata.GetContextsByTypeResponse.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionTypesByIDResponse} returns this + * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this */ -proto.ml_metadata.GetExecutionTypesByIDResponse.prototype.clearExecutionTypesList = function() { - return this.setExecutionTypesList([]); +proto.ml_metadata.GetContextsByTypeResponse.prototype.clearContextsList = function() { + return this.setContextsList([]); +}; + + +/** + * optional string next_page_token = 2; + * @return {string} + */ +proto.ml_metadata.GetContextsByTypeResponse.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this + */ +proto.ml_metadata.GetContextsByTypeResponse.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this + */ +proto.ml_metadata.GetContextsByTypeResponse.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetContextsByTypeResponse.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -14947,8 +21349,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextByTypeAndNameRequest.toObject(opt_includeInstance, this); }; @@ -14957,14 +21359,15 @@ proto.ml_metadata.GetContextTypeRequest.prototype.toObject = function(opt_includ * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextByTypeAndNameRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextByTypeAndNameRequest.toObject = function(includeInstance, msg) { var f, obj = { typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + contextName: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -14979,23 +21382,23 @@ proto.ml_metadata.GetContextTypeRequest.toObject = function(includeInstance, msg /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextTypeRequest} + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} */ -proto.ml_metadata.GetContextTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextTypeRequest; - return proto.ml_metadata.GetContextTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextByTypeAndNameRequest; + return proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextByTypeAndNameRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextTypeRequest} + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} */ -proto.ml_metadata.GetContextTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -15006,11 +21409,15 @@ proto.ml_metadata.GetContextTypeRequest.deserializeBinaryFromReader = function(m var value = /** @type {string} */ (reader.readString()); msg.setTypeName(value); break; - case 2: + case 3: var value = /** @type {string} */ (reader.readString()); msg.setTypeVersion(value); break; - case 3: + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setContextName(value); + break; + case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -15028,9 +21435,9 @@ proto.ml_metadata.GetContextTypeRequest.deserializeBinaryFromReader = function(m * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextByTypeAndNameRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -15038,11 +21445,11 @@ proto.ml_metadata.GetContextTypeRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextTypeRequest} message + * @param {!proto.ml_metadata.GetContextByTypeAndNameRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextByTypeAndNameRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; f = /** @type {string} */ (jspb.Message.getField(message, 1)); if (f != null) { @@ -15051,6 +21458,13 @@ proto.ml_metadata.GetContextTypeRequest.serializeBinaryToWriter = function(messa f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 3)); + if (f != null) { + writer.writeString( + 3, + f + ); + } f = /** @type {string} */ (jspb.Message.getField(message, 2)); if (f != null) { writer.writeString( @@ -15061,7 +21475,7 @@ proto.ml_metadata.GetContextTypeRequest.serializeBinaryToWriter = function(messa f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 3, + 4, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -15073,25 +21487,25 @@ proto.ml_metadata.GetContextTypeRequest.serializeBinaryToWriter = function(messa * optional string type_name = 1; * @return {string} */ -proto.ml_metadata.GetContextTypeRequest.prototype.getTypeName = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getTypeName = function() { return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); }; /** * @param {string} value - * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetContextTypeRequest.prototype.setTypeName = function(value) { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setTypeName = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetContextTypeRequest.prototype.clearTypeName = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTypeName = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -15100,34 +21514,70 @@ proto.ml_metadata.GetContextTypeRequest.prototype.clearTypeName = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextTypeRequest.prototype.hasTypeName = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasTypeName = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional string type_version = 2; + * optional string type_version = 3; * @return {string} */ -proto.ml_metadata.GetContextTypeRequest.prototype.getTypeVersion = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getTypeVersion = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this + */ +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setTypeVersion = function(value) { + return jspb.Message.setField(this, 3, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this + */ +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTypeVersion = function() { + return jspb.Message.setField(this, 3, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasTypeVersion = function() { + return jspb.Message.getField(this, 3) != null; +}; + + +/** + * optional string context_name = 2; + * @return {string} + */ +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getContextName = function() { return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** * @param {string} value - * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetContextTypeRequest.prototype.setTypeVersion = function(value) { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setContextName = function(value) { return jspb.Message.setField(this, 2, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetContextTypeRequest.prototype.clearTypeVersion = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearContextName = function() { return jspb.Message.setField(this, 2, undefined); }; @@ -15136,35 +21586,35 @@ proto.ml_metadata.GetContextTypeRequest.prototype.clearTypeVersion = function() * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextTypeRequest.prototype.hasTypeVersion = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasContextName = function() { return jspb.Message.getField(this, 2) != null; }; /** - * optional TransactionOptions transaction_options = 3; + * optional TransactionOptions transaction_options = 4; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextTypeRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetContextTypeRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 4, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this */ -proto.ml_metadata.GetContextTypeRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -15173,8 +21623,8 @@ proto.ml_metadata.GetContextTypeRequest.prototype.clearTransactionOptions = func * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextTypeRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 4) != null; }; @@ -15194,8 +21644,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextByTypeAndNameResponse.toObject(opt_includeInstance, this); }; @@ -15204,13 +21654,13 @@ proto.ml_metadata.GetContextTypeResponse.prototype.toObject = function(opt_inclu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextByTypeAndNameResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextByTypeAndNameResponse.toObject = function(includeInstance, msg) { var f, obj = { - contextType: (f = msg.getContextType()) && ml_metadata_proto_metadata_store_pb.ContextType.toObject(includeInstance, f) + context: (f = msg.getContext()) && ml_metadata_proto_metadata_store_pb.Context.toObject(includeInstance, f) }; if (includeInstance) { @@ -15224,23 +21674,23 @@ proto.ml_metadata.GetContextTypeResponse.toObject = function(includeInstance, ms /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextTypeResponse} + * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} */ -proto.ml_metadata.GetContextTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextTypeResponse; - return proto.ml_metadata.GetContextTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextByTypeAndNameResponse; + return proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextByTypeAndNameResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextTypeResponse} + * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} */ -proto.ml_metadata.GetContextTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -15248,9 +21698,9 @@ proto.ml_metadata.GetContextTypeResponse.deserializeBinaryFromReader = function( var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ContextType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); - msg.setContextType(value); + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.setContext(value); break; default: reader.skipField(); @@ -15265,9 +21715,9 @@ proto.ml_metadata.GetContextTypeResponse.deserializeBinaryFromReader = function( * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextByTypeAndNameResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -15275,48 +21725,48 @@ proto.ml_metadata.GetContextTypeResponse.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextTypeResponse} message + * @param {!proto.ml_metadata.GetContextByTypeAndNameResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextByTypeAndNameResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextType(); + f = message.getContext(); if (f != null) { writer.writeMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter ); } }; /** - * optional ContextType context_type = 1; - * @return {?proto.ml_metadata.ContextType} + * optional Context context = 1; + * @return {?proto.ml_metadata.Context} */ -proto.ml_metadata.GetContextTypeResponse.prototype.getContextType = function() { - return /** @type{?proto.ml_metadata.ContextType} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); +proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.getContext = function() { + return /** @type{?proto.ml_metadata.Context} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * @param {?proto.ml_metadata.ContextType|undefined} value - * @return {!proto.ml_metadata.GetContextTypeResponse} returns this + * @param {?proto.ml_metadata.Context|undefined} value + * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} returns this */ -proto.ml_metadata.GetContextTypeResponse.prototype.setContextType = function(value) { +proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.setContext = function(value) { return jspb.Message.setWrapperField(this, 1, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextTypeResponse} returns this + * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} returns this */ -proto.ml_metadata.GetContextTypeResponse.prototype.clearContextType = function() { - return this.setContextType(undefined); +proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.clearContext = function() { + return this.setContext(undefined); }; @@ -15324,7 +21774,7 @@ proto.ml_metadata.GetContextTypeResponse.prototype.clearContextType = function() * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextTypeResponse.prototype.hasContextType = function() { +proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.hasContext = function() { return jspb.Message.getField(this, 1) != null; }; @@ -15335,7 +21785,7 @@ proto.ml_metadata.GetContextTypeResponse.prototype.hasContextType = function() { * @private {!Array} * @const */ -proto.ml_metadata.GetContextTypesByIDRequest.repeatedFields_ = [1]; +proto.ml_metadata.GetContextsByIDRequest.repeatedFields_ = [1]; @@ -15352,8 +21802,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextTypesByIDRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByIDRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByIDRequest.toObject(opt_includeInstance, this); }; @@ -15362,13 +21812,13 @@ proto.ml_metadata.GetContextTypesByIDRequest.prototype.toObject = function(opt_i * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextTypesByIDRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByIDRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesByIDRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByIDRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + contextIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -15383,23 +21833,23 @@ proto.ml_metadata.GetContextTypesByIDRequest.toObject = function(includeInstance /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextTypesByIDRequest} + * @return {!proto.ml_metadata.GetContextsByIDRequest} */ -proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByIDRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextTypesByIDRequest; - return proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByIDRequest; + return proto.ml_metadata.GetContextsByIDRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextTypesByIDRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByIDRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextTypesByIDRequest} + * @return {!proto.ml_metadata.GetContextsByIDRequest} */ -proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByIDRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -15409,7 +21859,7 @@ proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinaryFromReader = funct case 1: var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); for (var i = 0; i < values.length; i++) { - msg.addTypeIds(values[i]); + msg.addContextIds(values[i]); } break; case 2: @@ -15430,9 +21880,9 @@ proto.ml_metadata.GetContextTypesByIDRequest.deserializeBinaryFromReader = funct * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByIDRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextTypesByIDRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByIDRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -15440,13 +21890,13 @@ proto.ml_metadata.GetContextTypesByIDRequest.prototype.serializeBinary = functio /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextTypesByIDRequest} message + * @param {!proto.ml_metadata.GetContextsByIDRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesByIDRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByIDRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getTypeIdsList(); + f = message.getContextIdsList(); if (f.length > 0) { writer.writeRepeatedInt64( 1, @@ -15465,19 +21915,19 @@ proto.ml_metadata.GetContextTypesByIDRequest.serializeBinaryToWriter = function( /** - * repeated int64 type_ids = 1; + * repeated int64 context_ids = 1; * @return {!Array} */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.getTypeIdsList = function() { +proto.ml_metadata.GetContextsByIDRequest.prototype.getContextIdsList = function() { return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** * @param {!Array} value - * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.setTypeIdsList = function(value) { +proto.ml_metadata.GetContextsByIDRequest.prototype.setContextIdsList = function(value) { return jspb.Message.setField(this, 1, value || []); }; @@ -15485,19 +21935,19 @@ proto.ml_metadata.GetContextTypesByIDRequest.prototype.setTypeIdsList = function /** * @param {number} value * @param {number=} opt_index - * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.addTypeIds = function(value, opt_index) { +proto.ml_metadata.GetContextsByIDRequest.prototype.addContextIds = function(value, opt_index) { return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.clearTypeIdsList = function() { - return this.setTypeIdsList([]); +proto.ml_metadata.GetContextsByIDRequest.prototype.clearContextIdsList = function() { + return this.setContextIdsList([]); }; @@ -15505,7 +21955,7 @@ proto.ml_metadata.GetContextTypesByIDRequest.prototype.clearTypeIdsList = functi * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextsByIDRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -15513,18 +21963,18 @@ proto.ml_metadata.GetContextTypesByIDRequest.prototype.getTransactionOptions = f /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetContextsByIDRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextTypesByIDRequest} returns this + * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextsByIDRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -15533,7 +21983,7 @@ proto.ml_metadata.GetContextTypesByIDRequest.prototype.clearTransactionOptions = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextTypesByIDRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetContextsByIDRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -15544,7 +21994,7 @@ proto.ml_metadata.GetContextTypesByIDRequest.prototype.hasTransactionOptions = f * @private {!Array} * @const */ -proto.ml_metadata.GetContextTypesByIDResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetContextsByIDResponse.repeatedFields_ = [1]; @@ -15561,8 +22011,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextTypesByIDResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextTypesByIDResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByIDResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByIDResponse.toObject(opt_includeInstance, this); }; @@ -15571,14 +22021,14 @@ proto.ml_metadata.GetContextTypesByIDResponse.prototype.toObject = function(opt_ * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextTypesByIDResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByIDResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesByIDResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByIDResponse.toObject = function(includeInstance, msg) { var f, obj = { - contextTypesList: jspb.Message.toObjectList(msg.getContextTypesList(), - ml_metadata_proto_metadata_store_pb.ContextType.toObject, includeInstance) + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) }; if (includeInstance) { @@ -15592,23 +22042,23 @@ proto.ml_metadata.GetContextTypesByIDResponse.toObject = function(includeInstanc /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextTypesByIDResponse} + * @return {!proto.ml_metadata.GetContextsByIDResponse} */ -proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByIDResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextTypesByIDResponse; - return proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByIDResponse; + return proto.ml_metadata.GetContextsByIDResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextTypesByIDResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByIDResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextTypesByIDResponse} + * @return {!proto.ml_metadata.GetContextsByIDResponse} */ -proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByIDResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -15616,9 +22066,9 @@ proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinaryFromReader = func var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ContextType; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ContextType.deserializeBinaryFromReader); - msg.addContextTypes(value); + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addContexts(value); break; default: reader.skipField(); @@ -15633,9 +22083,9 @@ proto.ml_metadata.GetContextTypesByIDResponse.deserializeBinaryFromReader = func * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextTypesByIDResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByIDResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextTypesByIDResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByIDResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -15643,58 +22093,58 @@ proto.ml_metadata.GetContextTypesByIDResponse.prototype.serializeBinary = functi /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextTypesByIDResponse} message + * @param {!proto.ml_metadata.GetContextsByIDResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextTypesByIDResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByIDResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextTypesList(); + f = message.getContextsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.ContextType.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter ); } }; /** - * repeated ContextType context_types = 1; - * @return {!Array} + * repeated Context contexts = 1; + * @return {!Array} */ -proto.ml_metadata.GetContextTypesByIDResponse.prototype.getContextTypesList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.ContextType, 1)); +proto.ml_metadata.GetContextsByIDResponse.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetContextTypesByIDResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetContextsByIDResponse} returns this */ -proto.ml_metadata.GetContextTypesByIDResponse.prototype.setContextTypesList = function(value) { +proto.ml_metadata.GetContextsByIDResponse.prototype.setContextsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.ContextType=} opt_value + * @param {!proto.ml_metadata.Context=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.ContextType} + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetContextTypesByIDResponse.prototype.addContextTypes = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.ContextType, opt_index); +proto.ml_metadata.GetContextsByIDResponse.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextTypesByIDResponse} returns this + * @return {!proto.ml_metadata.GetContextsByIDResponse} returns this */ -proto.ml_metadata.GetContextTypesByIDResponse.prototype.clearContextTypesList = function() { - return this.setContextTypesList([]); +proto.ml_metadata.GetContextsByIDResponse.prototype.clearContextsList = function() { + return this.setContextsList([]); }; @@ -15714,8 +22164,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByArtifactRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByArtifactRequest.toObject(opt_includeInstance, this); }; @@ -15724,13 +22174,13 @@ proto.ml_metadata.GetContextsRequest.prototype.toObject = function(opt_includeIn * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByArtifactRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByArtifactRequest.toObject = function(includeInstance, msg) { var f, obj = { - options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + artifactId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -15745,23 +22195,23 @@ proto.ml_metadata.GetContextsRequest.toObject = function(includeInstance, msg) { /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsRequest} + * @return {!proto.ml_metadata.GetContextsByArtifactRequest} */ -proto.ml_metadata.GetContextsRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsRequest; - return proto.ml_metadata.GetContextsRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByArtifactRequest; + return proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByArtifactRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsRequest} + * @return {!proto.ml_metadata.GetContextsByArtifactRequest} */ -proto.ml_metadata.GetContextsRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -15769,9 +22219,8 @@ proto.ml_metadata.GetContextsRequest.deserializeBinaryFromReader = function(msg, var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); - msg.setOptions(value); + var value = /** @type {number} */ (reader.readInt64()); + msg.setArtifactId(value); break; case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; @@ -15791,9 +22240,9 @@ proto.ml_metadata.GetContextsRequest.deserializeBinaryFromReader = function(msg, * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByArtifactRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByArtifactRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -15801,18 +22250,17 @@ proto.ml_metadata.GetContextsRequest.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsRequest} message + * @param {!proto.ml_metadata.GetContextsByArtifactRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByArtifactRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getOptions(); + f = /** @type {number} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeMessage( + writer.writeInt64( 1, - f, - ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter + f ); } f = message.getTransactionOptions(); @@ -15827,30 +22275,29 @@ proto.ml_metadata.GetContextsRequest.serializeBinaryToWriter = function(message, /** - * optional ListOperationOptions options = 1; - * @return {?proto.ml_metadata.ListOperationOptions} + * optional int64 artifact_id = 1; + * @return {number} */ -proto.ml_metadata.GetContextsRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 1)); +proto.ml_metadata.GetContextsByArtifactRequest.prototype.getArtifactId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetContextsRequest} returns this -*/ -proto.ml_metadata.GetContextsRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 1, value); + * @param {number} value + * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this + */ +proto.ml_metadata.GetContextsByArtifactRequest.prototype.setArtifactId = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextsRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this */ -proto.ml_metadata.GetContextsRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); +proto.ml_metadata.GetContextsByArtifactRequest.prototype.clearArtifactId = function() { + return jspb.Message.setField(this, 1, undefined); }; @@ -15858,7 +22305,7 @@ proto.ml_metadata.GetContextsRequest.prototype.clearOptions = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextsRequest.prototype.hasOptions = function() { +proto.ml_metadata.GetContextsByArtifactRequest.prototype.hasArtifactId = function() { return jspb.Message.getField(this, 1) != null; }; @@ -15867,7 +22314,7 @@ proto.ml_metadata.GetContextsRequest.prototype.hasOptions = function() { * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextsRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextsByArtifactRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -15875,18 +22322,18 @@ proto.ml_metadata.GetContextsRequest.prototype.getTransactionOptions = function( /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextsRequest} returns this + * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this */ -proto.ml_metadata.GetContextsRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetContextsByArtifactRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextsRequest} returns this + * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this */ -proto.ml_metadata.GetContextsRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextsByArtifactRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -15895,7 +22342,7 @@ proto.ml_metadata.GetContextsRequest.prototype.clearTransactionOptions = functio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextsRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetContextsByArtifactRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -15906,7 +22353,7 @@ proto.ml_metadata.GetContextsRequest.prototype.hasTransactionOptions = function( * @private {!Array} * @const */ -proto.ml_metadata.GetContextsResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetContextsByArtifactResponse.repeatedFields_ = [1]; @@ -15923,8 +22370,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByArtifactResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByArtifactResponse.toObject(opt_includeInstance, this); }; @@ -15933,15 +22380,14 @@ proto.ml_metadata.GetContextsResponse.prototype.toObject = function(opt_includeI * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByArtifactResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByArtifactResponse.toObject = function(includeInstance, msg) { var f, obj = { contextsList: jspb.Message.toObjectList(msg.getContextsList(), - ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) }; if (includeInstance) { @@ -15955,23 +22401,23 @@ proto.ml_metadata.GetContextsResponse.toObject = function(includeInstance, msg) /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsResponse} + * @return {!proto.ml_metadata.GetContextsByArtifactResponse} */ -proto.ml_metadata.GetContextsResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByArtifactResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsResponse; - return proto.ml_metadata.GetContextsResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByArtifactResponse; + return proto.ml_metadata.GetContextsByArtifactResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByArtifactResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsResponse} + * @return {!proto.ml_metadata.GetContextsByArtifactResponse} */ -proto.ml_metadata.GetContextsResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByArtifactResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -15983,10 +22429,6 @@ proto.ml_metadata.GetContextsResponse.deserializeBinaryFromReader = function(msg reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); msg.addContexts(value); break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); - break; default: reader.skipField(); break; @@ -16000,9 +22442,9 @@ proto.ml_metadata.GetContextsResponse.deserializeBinaryFromReader = function(msg * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByArtifactResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByArtifactResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -16010,11 +22452,11 @@ proto.ml_metadata.GetContextsResponse.prototype.serializeBinary = function() { /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsResponse} message + * @param {!proto.ml_metadata.GetContextsByArtifactResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByArtifactResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; f = message.getContextsList(); if (f.length > 0) { @@ -16024,13 +22466,6 @@ proto.ml_metadata.GetContextsResponse.serializeBinaryToWriter = function(message ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } }; @@ -16038,7 +22473,7 @@ proto.ml_metadata.GetContextsResponse.serializeBinaryToWriter = function(message * repeated Context contexts = 1; * @return {!Array} */ -proto.ml_metadata.GetContextsResponse.prototype.getContextsList = function() { +proto.ml_metadata.GetContextsByArtifactResponse.prototype.getContextsList = function() { return /** @type{!Array} */ ( jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; @@ -16046,9 +22481,9 @@ proto.ml_metadata.GetContextsResponse.prototype.getContextsList = function() { /** * @param {!Array} value - * @return {!proto.ml_metadata.GetContextsResponse} returns this + * @return {!proto.ml_metadata.GetContextsByArtifactResponse} returns this */ -proto.ml_metadata.GetContextsResponse.prototype.setContextsList = function(value) { +proto.ml_metadata.GetContextsByArtifactResponse.prototype.setContextsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; @@ -16058,56 +22493,20 @@ proto.ml_metadata.GetContextsResponse.prototype.setContextsList = function(value * @param {number=} opt_index * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetContextsResponse.prototype.addContexts = function(opt_value, opt_index) { +proto.ml_metadata.GetContextsByArtifactResponse.prototype.addContexts = function(opt_value, opt_index) { return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextsResponse} returns this + * @return {!proto.ml_metadata.GetContextsByArtifactResponse} returns this */ -proto.ml_metadata.GetContextsResponse.prototype.clearContextsList = function() { +proto.ml_metadata.GetContextsByArtifactResponse.prototype.clearContextsList = function() { return this.setContextsList([]); }; -/** - * optional string next_page_token = 2; - * @return {string} - */ -proto.ml_metadata.GetContextsResponse.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetContextsResponse} returns this - */ -proto.ml_metadata.GetContextsResponse.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 2, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextsResponse} returns this - */ -proto.ml_metadata.GetContextsResponse.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 2, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetContextsResponse.prototype.hasNextPageToken = function() { - return jspb.Message.getField(this, 2) != null; -}; - - @@ -16124,8 +22523,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByTypeRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByExecutionRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByExecutionRequest.toObject(opt_includeInstance, this); }; @@ -16134,15 +22533,13 @@ proto.ml_metadata.GetContextsByTypeRequest.prototype.toObject = function(opt_inc * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByTypeRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByExecutionRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByTypeRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByExecutionRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), - typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, + executionId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -16157,23 +22554,23 @@ proto.ml_metadata.GetContextsByTypeRequest.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByTypeRequest} + * @return {!proto.ml_metadata.GetContextsByExecutionRequest} */ -proto.ml_metadata.GetContextsByTypeRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByTypeRequest; - return proto.ml_metadata.GetContextsByTypeRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByExecutionRequest; + return proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByTypeRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByExecutionRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByTypeRequest} + * @return {!proto.ml_metadata.GetContextsByExecutionRequest} */ -proto.ml_metadata.GetContextsByTypeRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -16181,19 +22578,10 @@ proto.ml_metadata.GetContextsByTypeRequest.deserializeBinaryFromReader = functio var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); + var value = /** @type {number} */ (reader.readInt64()); + msg.setExecutionId(value); break; case 2: - var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); - msg.setOptions(value); - break; - case 3: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); - break; - case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -16211,9 +22599,9 @@ proto.ml_metadata.GetContextsByTypeRequest.deserializeBinaryFromReader = functio * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByExecutionRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByTypeRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByExecutionRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -16221,38 +22609,23 @@ proto.ml_metadata.GetContextsByTypeRequest.prototype.serializeBinary = function( /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByTypeRequest} message + * @param {!proto.ml_metadata.GetContextsByExecutionRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByTypeRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByExecutionRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); + f = /** @type {number} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeString( + writer.writeInt64( 1, f ); } - f = message.getOptions(); - if (f != null) { - writer.writeMessage( - 2, - f, - ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeString( - 3, - f - ); - } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 4, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -16261,28 +22634,28 @@ proto.ml_metadata.GetContextsByTypeRequest.serializeBinaryToWriter = function(me /** - * optional string type_name = 1; - * @return {string} + * optional int64 execution_id = 1; + * @return {number} */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +proto.ml_metadata.GetContextsByExecutionRequest.prototype.getExecutionId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this + * @param {number} value + * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.setTypeName = function(value) { +proto.ml_metadata.GetContextsByExecutionRequest.prototype.setExecutionId = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTypeName = function() { +proto.ml_metadata.GetContextsByExecutionRequest.prototype.clearExecutionId = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -16291,108 +22664,35 @@ proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTypeName = function() * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.hasTypeName = function() { +proto.ml_metadata.GetContextsByExecutionRequest.prototype.hasExecutionId = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional ListOperationOptions options = 2; - * @return {?proto.ml_metadata.ListOperationOptions} - */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 2)); -}; - - -/** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this -*/ -proto.ml_metadata.GetContextsByTypeRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this - */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 2) != null; -}; - - -/** - * optional string type_version = 3; - * @return {string} - */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this - */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 3, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this - */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 3, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.hasTypeVersion = function() { - return jspb.Message.getField(this, 3) != null; -}; - - -/** - * optional TransactionOptions transaction_options = 4; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetContextsByExecutionRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); +proto.ml_metadata.GetContextsByExecutionRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextsByTypeRequest} returns this + * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetContextsByExecutionRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -16401,8 +22701,8 @@ proto.ml_metadata.GetContextsByTypeRequest.prototype.clearTransactionOptions = f * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextsByTypeRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetContextsByExecutionRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -16412,7 +22712,7 @@ proto.ml_metadata.GetContextsByTypeRequest.prototype.hasTransactionOptions = fun * @private {!Array} * @const */ -proto.ml_metadata.GetContextsByTypeResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetContextsByExecutionResponse.repeatedFields_ = [1]; @@ -16429,8 +22729,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByTypeResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetContextsByExecutionResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetContextsByExecutionResponse.toObject(opt_includeInstance, this); }; @@ -16439,15 +22739,14 @@ proto.ml_metadata.GetContextsByTypeResponse.prototype.toObject = function(opt_in * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByTypeResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetContextsByExecutionResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByTypeResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetContextsByExecutionResponse.toObject = function(includeInstance, msg) { var f, obj = { contextsList: jspb.Message.toObjectList(msg.getContextsList(), - ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) }; if (includeInstance) { @@ -16461,23 +22760,23 @@ proto.ml_metadata.GetContextsByTypeResponse.toObject = function(includeInstance, /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByTypeResponse} + * @return {!proto.ml_metadata.GetContextsByExecutionResponse} */ -proto.ml_metadata.GetContextsByTypeResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByTypeResponse; - return proto.ml_metadata.GetContextsByTypeResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetContextsByExecutionResponse; + return proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByTypeResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetContextsByExecutionResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByTypeResponse} + * @return {!proto.ml_metadata.GetContextsByExecutionResponse} */ -proto.ml_metadata.GetContextsByTypeResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -16489,10 +22788,6 @@ proto.ml_metadata.GetContextsByTypeResponse.deserializeBinaryFromReader = functi reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); msg.addContexts(value); break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); - break; default: reader.skipField(); break; @@ -16506,9 +22801,9 @@ proto.ml_metadata.GetContextsByTypeResponse.deserializeBinaryFromReader = functi * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetContextsByExecutionResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByTypeResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetContextsByExecutionResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -16516,11 +22811,11 @@ proto.ml_metadata.GetContextsByTypeResponse.prototype.serializeBinary = function /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByTypeResponse} message + * @param {!proto.ml_metadata.GetContextsByExecutionResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByTypeResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetContextsByExecutionResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; f = message.getContextsList(); if (f.length > 0) { @@ -16530,13 +22825,6 @@ proto.ml_metadata.GetContextsByTypeResponse.serializeBinaryToWriter = function(m ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } }; @@ -16544,7 +22832,7 @@ proto.ml_metadata.GetContextsByTypeResponse.serializeBinaryToWriter = function(m * repeated Context contexts = 1; * @return {!Array} */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.getContextsList = function() { +proto.ml_metadata.GetContextsByExecutionResponse.prototype.getContextsList = function() { return /** @type{!Array} */ ( jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; @@ -16552,65 +22840,29 @@ proto.ml_metadata.GetContextsByTypeResponse.prototype.getContextsList = function /** * @param {!Array} value - * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this + * @return {!proto.ml_metadata.GetContextsByExecutionResponse} returns this */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.setContextsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); -}; - - -/** - * @param {!proto.ml_metadata.Context=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.Context} - */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.addContexts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); -}; - - -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this - */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.clearContextsList = function() { - return this.setContextsList([]); -}; - - -/** - * optional string next_page_token = 2; - * @return {string} - */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this - */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 2, value); +proto.ml_metadata.GetContextsByExecutionResponse.prototype.setContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextsByTypeResponse} returns this + * @param {!proto.ml_metadata.Context=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 2, undefined); +proto.ml_metadata.GetContextsByExecutionResponse.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetContextsByExecutionResponse} returns this */ -proto.ml_metadata.GetContextsByTypeResponse.prototype.hasNextPageToken = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetContextsByExecutionResponse.prototype.clearContextsList = function() { + return this.setContextsList([]); }; @@ -16630,8 +22882,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextByTypeAndNameRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetParentContextsByContextRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetParentContextsByContextRequest.toObject(opt_includeInstance, this); }; @@ -16640,15 +22892,13 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.toObject = function(o * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextByTypeAndNameRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetParentContextsByContextRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextByTypeAndNameRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetParentContextsByContextRequest.toObject = function(includeInstance, msg) { var f, obj = { - typeName: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - typeVersion: (f = jspb.Message.getField(msg, 3)) == null ? undefined : f, - contextName: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + contextId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -16663,23 +22913,23 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.toObject = function(includeInst /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} + * @return {!proto.ml_metadata.GetParentContextsByContextRequest} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetParentContextsByContextRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextByTypeAndNameRequest; - return proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetParentContextsByContextRequest; + return proto.ml_metadata.GetParentContextsByContextRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextByTypeAndNameRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetParentContextsByContextRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} + * @return {!proto.ml_metadata.GetParentContextsByContextRequest} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetParentContextsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -16687,18 +22937,10 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinaryFromReader = f var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeName(value); - break; - case 3: - var value = /** @type {string} */ (reader.readString()); - msg.setTypeVersion(value); + var value = /** @type {number} */ (reader.readInt64()); + msg.setContextId(value); break; case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setContextName(value); - break; - case 4: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -16716,9 +22958,9 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.deserializeBinaryFromReader = f * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetParentContextsByContextRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextByTypeAndNameRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetParentContextsByContextRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -16726,37 +22968,23 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.serializeBinary = fun /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextByTypeAndNameRequest} message + * @param {!proto.ml_metadata.GetParentContextsByContextRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextByTypeAndNameRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetParentContextsByContextRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {string} */ (jspb.Message.getField(message, 1)); + f = /** @type {number} */ (jspb.Message.getField(message, 1)); if (f != null) { - writer.writeString( + writer.writeInt64( 1, f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeString( - 3, - f - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 4, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -16765,28 +22993,28 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.serializeBinaryToWriter = funct /** - * optional string type_name = 1; - * @return {string} + * optional int64 context_id = 1; + * @return {number} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getTypeName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +proto.ml_metadata.GetParentContextsByContextRequest.prototype.getContextId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** - * @param {string} value - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this + * @param {number} value + * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setTypeName = function(value) { +proto.ml_metadata.GetParentContextsByContextRequest.prototype.setContextId = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this + * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTypeName = function() { +proto.ml_metadata.GetParentContextsByContextRequest.prototype.clearContextId = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -16795,107 +23023,35 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTypeName = funct * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasTypeName = function() { +proto.ml_metadata.GetParentContextsByContextRequest.prototype.hasContextId = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional string type_version = 3; - * @return {string} - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getTypeVersion = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setTypeVersion = function(value) { - return jspb.Message.setField(this, 3, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTypeVersion = function() { - return jspb.Message.setField(this, 3, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasTypeVersion = function() { - return jspb.Message.getField(this, 3) != null; -}; - - -/** - * optional string context_name = 2; - * @return {string} - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getContextName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setContextName = function(value) { - return jspb.Message.setField(this, 2, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearContextName = function() { - return jspb.Message.setField(this, 2, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasContextName = function() { - return jspb.Message.getField(this, 2) != null; -}; - - -/** - * optional TransactionOptions transaction_options = 4; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetParentContextsByContextRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 4)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this + * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 4, value); +proto.ml_metadata.GetParentContextsByContextRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextByTypeAndNameRequest} returns this + * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetParentContextsByContextRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -16904,12 +23060,19 @@ proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.clearTransactionOptio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextByTypeAndNameRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 4) != null; +proto.ml_metadata.GetParentContextsByContextRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetParentContextsByContextResponse.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -16925,8 +23088,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextByTypeAndNameResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetParentContextsByContextResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetParentContextsByContextResponse.toObject(opt_includeInstance, this); }; @@ -16935,13 +23098,14 @@ proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.toObject = function( * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextByTypeAndNameResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetParentContextsByContextResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextByTypeAndNameResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetParentContextsByContextResponse.toObject = function(includeInstance, msg) { var f, obj = { - context: (f = msg.getContext()) && ml_metadata_proto_metadata_store_pb.Context.toObject(includeInstance, f) + contextsList: jspb.Message.toObjectList(msg.getContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) }; if (includeInstance) { @@ -16955,23 +23119,23 @@ proto.ml_metadata.GetContextByTypeAndNameResponse.toObject = function(includeIns /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} + * @return {!proto.ml_metadata.GetParentContextsByContextResponse} */ -proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextByTypeAndNameResponse; - return proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetParentContextsByContextResponse; + return proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextByTypeAndNameResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetParentContextsByContextResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} + * @return {!proto.ml_metadata.GetParentContextsByContextResponse} */ -proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -16981,7 +23145,7 @@ proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinaryFromReader = case 1: var value = new ml_metadata_proto_metadata_store_pb.Context; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); - msg.setContext(value); + msg.addContexts(value); break; default: reader.skipField(); @@ -16996,9 +23160,9 @@ proto.ml_metadata.GetContextByTypeAndNameResponse.deserializeBinaryFromReader = * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetParentContextsByContextResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextByTypeAndNameResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetParentContextsByContextResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -17006,15 +23170,15 @@ proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.serializeBinary = fu /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextByTypeAndNameResponse} message + * @param {!proto.ml_metadata.GetParentContextsByContextResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextByTypeAndNameResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetParentContextsByContextResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContext(); - if (f != null) { - writer.writeMessage( + f = message.getContextsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, f, ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter @@ -17024,50 +23188,44 @@ proto.ml_metadata.GetContextByTypeAndNameResponse.serializeBinaryToWriter = func /** - * optional Context context = 1; - * @return {?proto.ml_metadata.Context} + * repeated Context contexts = 1; + * @return {!Array} */ -proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.getContext = function() { - return /** @type{?proto.ml_metadata.Context} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); +proto.ml_metadata.GetParentContextsByContextResponse.prototype.getContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * @param {?proto.ml_metadata.Context|undefined} value - * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetParentContextsByContextResponse} returns this */ -proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.setContext = function(value) { - return jspb.Message.setWrapperField(this, 1, value); +proto.ml_metadata.GetParentContextsByContextResponse.prototype.setContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextByTypeAndNameResponse} returns this + * @param {!proto.ml_metadata.Context=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.clearContext = function() { - return this.setContext(undefined); +proto.ml_metadata.GetParentContextsByContextResponse.prototype.addContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetParentContextsByContextResponse} returns this */ -proto.ml_metadata.GetContextByTypeAndNameResponse.prototype.hasContext = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetParentContextsByContextResponse.prototype.clearContextsList = function() { + return this.setContextsList([]); }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetContextsByIDRequest.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -17083,8 +23241,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByIDRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByIDRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetChildrenContextsByContextRequest.toObject(opt_includeInstance, this); }; @@ -17093,13 +23251,13 @@ proto.ml_metadata.GetContextsByIDRequest.prototype.toObject = function(opt_inclu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByIDRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetChildrenContextsByContextRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByIDRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetChildrenContextsByContextRequest.toObject = function(includeInstance, msg) { var f, obj = { - contextIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + contextId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -17114,23 +23272,23 @@ proto.ml_metadata.GetContextsByIDRequest.toObject = function(includeInstance, ms /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByIDRequest} + * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} */ -proto.ml_metadata.GetContextsByIDRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByIDRequest; - return proto.ml_metadata.GetContextsByIDRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetChildrenContextsByContextRequest; + return proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByIDRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetChildrenContextsByContextRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByIDRequest} + * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} */ -proto.ml_metadata.GetContextsByIDRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -17138,10 +23296,8 @@ proto.ml_metadata.GetContextsByIDRequest.deserializeBinaryFromReader = function( var field = reader.getFieldNumber(); switch (field) { case 1: - var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); - for (var i = 0; i < values.length; i++) { - msg.addContextIds(values[i]); - } + var value = /** @type {number} */ (reader.readInt64()); + msg.setContextId(value); break; case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; @@ -17161,9 +23317,9 @@ proto.ml_metadata.GetContextsByIDRequest.deserializeBinaryFromReader = function( * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByIDRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByIDRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetChildrenContextsByContextRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -17171,15 +23327,15 @@ proto.ml_metadata.GetContextsByIDRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByIDRequest} message + * @param {!proto.ml_metadata.GetChildrenContextsByContextRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByIDRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetChildrenContextsByContextRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextIdsList(); - if (f.length > 0) { - writer.writeRepeatedInt64( + f = /** @type {number} */ (jspb.Message.getField(message, 1)); + if (f != null) { + writer.writeInt64( 1, f ); @@ -17196,39 +23352,38 @@ proto.ml_metadata.GetContextsByIDRequest.serializeBinaryToWriter = function(mess /** - * repeated int64 context_ids = 1; - * @return {!Array} + * optional int64 context_id = 1; + * @return {number} */ -proto.ml_metadata.GetContextsByIDRequest.prototype.getContextIdsList = function() { - return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.getContextId = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this + * @param {number} value + * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextsByIDRequest.prototype.setContextIdsList = function(value) { - return jspb.Message.setField(this, 1, value || []); +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.setContextId = function(value) { + return jspb.Message.setField(this, 1, value); }; /** - * @param {number} value - * @param {number=} opt_index - * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextsByIDRequest.prototype.addContextIds = function(value, opt_index) { - return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.clearContextId = function() { + return jspb.Message.setField(this, 1, undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetContextsByIDRequest.prototype.clearContextIdsList = function() { - return this.setContextIdsList([]); +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.hasContextId = function() { + return jspb.Message.getField(this, 1) != null; }; @@ -17236,7 +23391,7 @@ proto.ml_metadata.GetContextsByIDRequest.prototype.clearContextIdsList = functio * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextsByIDRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -17244,18 +23399,18 @@ proto.ml_metadata.GetContextsByIDRequest.prototype.getTransactionOptions = funct /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this + * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextsByIDRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextsByIDRequest} returns this + * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this */ -proto.ml_metadata.GetContextsByIDRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -17264,7 +23419,7 @@ proto.ml_metadata.GetContextsByIDRequest.prototype.clearTransactionOptions = fun * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextsByIDRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -17275,7 +23430,7 @@ proto.ml_metadata.GetContextsByIDRequest.prototype.hasTransactionOptions = funct * @private {!Array} * @const */ -proto.ml_metadata.GetContextsByIDResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetChildrenContextsByContextResponse.repeatedFields_ = [1]; @@ -17292,8 +23447,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByIDResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByIDResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetChildrenContextsByContextResponse.toObject(opt_includeInstance, this); }; @@ -17302,11 +23457,11 @@ proto.ml_metadata.GetContextsByIDResponse.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByIDResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetChildrenContextsByContextResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByIDResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetChildrenContextsByContextResponse.toObject = function(includeInstance, msg) { var f, obj = { contextsList: jspb.Message.toObjectList(msg.getContextsList(), ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) @@ -17323,23 +23478,23 @@ proto.ml_metadata.GetContextsByIDResponse.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByIDResponse} + * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} */ -proto.ml_metadata.GetContextsByIDResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByIDResponse; - return proto.ml_metadata.GetContextsByIDResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetChildrenContextsByContextResponse; + return proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByIDResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetChildrenContextsByContextResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByIDResponse} + * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} */ -proto.ml_metadata.GetContextsByIDResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -17364,9 +23519,9 @@ proto.ml_metadata.GetContextsByIDResponse.deserializeBinaryFromReader = function * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByIDResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByIDResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetChildrenContextsByContextResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -17374,11 +23529,11 @@ proto.ml_metadata.GetContextsByIDResponse.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByIDResponse} message + * @param {!proto.ml_metadata.GetChildrenContextsByContextResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByIDResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetChildrenContextsByContextResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; f = message.getContextsList(); if (f.length > 0) { @@ -17395,7 +23550,7 @@ proto.ml_metadata.GetContextsByIDResponse.serializeBinaryToWriter = function(mes * repeated Context contexts = 1; * @return {!Array} */ -proto.ml_metadata.GetContextsByIDResponse.prototype.getContextsList = function() { +proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.getContextsList = function() { return /** @type{!Array} */ ( jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; @@ -17403,9 +23558,9 @@ proto.ml_metadata.GetContextsByIDResponse.prototype.getContextsList = function() /** * @param {!Array} value - * @return {!proto.ml_metadata.GetContextsByIDResponse} returns this + * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} returns this */ -proto.ml_metadata.GetContextsByIDResponse.prototype.setContextsList = function(value) { +proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.setContextsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; @@ -17415,21 +23570,28 @@ proto.ml_metadata.GetContextsByIDResponse.prototype.setContextsList = function(v * @param {number=} opt_index * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetContextsByIDResponse.prototype.addContexts = function(opt_value, opt_index) { +proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.addContexts = function(opt_value, opt_index) { return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextsByIDResponse} returns this + * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} returns this */ -proto.ml_metadata.GetContextsByIDResponse.prototype.clearContextsList = function() { +proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.clearContextsList = function() { return this.setContextsList([]); }; +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.ml_metadata.GetParentContextsByContextsRequest.repeatedFields_ = [1]; + if (jspb.Message.GENERATE_TO_OBJECT) { @@ -17445,8 +23607,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByArtifactRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetParentContextsByContextsRequest.toObject(opt_includeInstance, this); }; @@ -17455,13 +23617,13 @@ proto.ml_metadata.GetContextsByArtifactRequest.prototype.toObject = function(opt * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByArtifactRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetParentContextsByContextsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByArtifactRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetParentContextsByContextsRequest.toObject = function(includeInstance, msg) { var f, obj = { - artifactId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + contextIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -17476,23 +23638,23 @@ proto.ml_metadata.GetContextsByArtifactRequest.toObject = function(includeInstan /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByArtifactRequest} + * @return {!proto.ml_metadata.GetParentContextsByContextsRequest} */ -proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetParentContextsByContextsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByArtifactRequest; - return proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetParentContextsByContextsRequest; + return proto.ml_metadata.GetParentContextsByContextsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByArtifactRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetParentContextsByContextsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByArtifactRequest} + * @return {!proto.ml_metadata.GetParentContextsByContextsRequest} */ -proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetParentContextsByContextsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -17500,8 +23662,10 @@ proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinaryFromReader = fun var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setArtifactId(value); + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addContextIds(values[i]); + } break; case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; @@ -17521,9 +23685,9 @@ proto.ml_metadata.GetContextsByArtifactRequest.deserializeBinaryFromReader = fun * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByArtifactRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetParentContextsByContextsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -17531,15 +23695,15 @@ proto.ml_metadata.GetContextsByArtifactRequest.prototype.serializeBinary = funct /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByArtifactRequest} message + * @param {!proto.ml_metadata.GetParentContextsByContextsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByArtifactRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetParentContextsByContextsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( + f = message.getContextIdsList(); + if (f.length > 0) { + writer.writePackedInt64( 1, f ); @@ -17556,38 +23720,39 @@ proto.ml_metadata.GetContextsByArtifactRequest.serializeBinaryToWriter = functio /** - * optional int64 artifact_id = 1; - * @return {number} + * repeated int64 context_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.getArtifactId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.getContextIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {number} value - * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetParentContextsByContextsRequest} returns this */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.setArtifactId = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.setContextIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this + * @param {number} value + * @param {number=} opt_index + * @return {!proto.ml_metadata.GetParentContextsByContextsRequest} returns this */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.clearArtifactId = function() { - return jspb.Message.setField(this, 1, undefined); +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.addContextIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetParentContextsByContextsRequest} returns this */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.hasArtifactId = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.clearContextIdsList = function() { + return this.setContextIdsList([]); }; @@ -17595,7 +23760,7 @@ proto.ml_metadata.GetContextsByArtifactRequest.prototype.hasArtifactId = functio * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -17603,18 +23768,18 @@ proto.ml_metadata.GetContextsByArtifactRequest.prototype.getTransactionOptions = /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this + * @return {!proto.ml_metadata.GetParentContextsByContextsRequest} returns this */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextsByArtifactRequest} returns this + * @return {!proto.ml_metadata.GetParentContextsByContextsRequest} returns this */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -17623,19 +23788,12 @@ proto.ml_metadata.GetContextsByArtifactRequest.prototype.clearTransactionOptions * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetContextsByArtifactRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetParentContextsByContextsRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetContextsByArtifactResponse.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -17651,8 +23809,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByArtifactResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByArtifactResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetParentContextsByContextsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetParentContextsByContextsResponse.toObject(opt_includeInstance, this); }; @@ -17661,14 +23819,13 @@ proto.ml_metadata.GetContextsByArtifactResponse.prototype.toObject = function(op * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByArtifactResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetParentContextsByContextsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByArtifactResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetParentContextsByContextsResponse.toObject = function(includeInstance, msg) { var f, obj = { - contextsList: jspb.Message.toObjectList(msg.getContextsList(), - ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) + contextsMap: (f = msg.getContextsMap()) ? f.toObject(includeInstance, proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.toObject) : [] }; if (includeInstance) { @@ -17682,33 +23839,34 @@ proto.ml_metadata.GetContextsByArtifactResponse.toObject = function(includeInsta /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByArtifactResponse} + * @return {!proto.ml_metadata.GetParentContextsByContextsResponse} */ -proto.ml_metadata.GetContextsByArtifactResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetParentContextsByContextsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByArtifactResponse; - return proto.ml_metadata.GetContextsByArtifactResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetParentContextsByContextsResponse; + return proto.ml_metadata.GetParentContextsByContextsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByArtifactResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetParentContextsByContextsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByArtifactResponse} + * @return {!proto.ml_metadata.GetParentContextsByContextsResponse} */ -proto.ml_metadata.GetContextsByArtifactResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetParentContextsByContextsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; - } - var field = reader.getFieldNumber(); - switch (field) { - case 1: - var value = new ml_metadata_proto_metadata_store_pb.Context; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); - msg.addContexts(value); + } + var field = reader.getFieldNumber(); + switch (field) { + case 2: + var value = msg.getContextsMap(); + reader.readMessage(value, function(message, reader) { + jspb.Map.deserializeBinary(message, reader, jspb.BinaryReader.prototype.readInt64, jspb.BinaryReader.prototype.readMessage, proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.deserializeBinaryFromReader, 0, new proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild()); + }); break; default: reader.skipField(); @@ -17723,9 +23881,9 @@ proto.ml_metadata.GetContextsByArtifactResponse.deserializeBinaryFromReader = fu * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByArtifactResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetParentContextsByContextsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByArtifactResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetParentContextsByContextsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -17733,61 +23891,26 @@ proto.ml_metadata.GetContextsByArtifactResponse.prototype.serializeBinary = func /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByArtifactResponse} message + * @param {!proto.ml_metadata.GetParentContextsByContextsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByArtifactResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetParentContextsByContextsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextsList(); - if (f.length > 0) { - writer.writeRepeatedMessage( - 1, - f, - ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter - ); + f = message.getContextsMap(true); + if (f && f.getLength() > 0) { + f.serializeBinary(2, writer, jspb.BinaryWriter.prototype.writeInt64, jspb.BinaryWriter.prototype.writeMessage, proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.serializeBinaryToWriter); } }; -/** - * repeated Context contexts = 1; - * @return {!Array} - */ -proto.ml_metadata.GetContextsByArtifactResponse.prototype.getContextsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.GetContextsByArtifactResponse} returns this -*/ -proto.ml_metadata.GetContextsByArtifactResponse.prototype.setContextsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); -}; - - -/** - * @param {!proto.ml_metadata.Context=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.Context} - */ -proto.ml_metadata.GetContextsByArtifactResponse.prototype.addContexts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); -}; - /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextsByArtifactResponse} returns this + * List of repeated fields within this message type. + * @private {!Array} + * @const */ -proto.ml_metadata.GetContextsByArtifactResponse.prototype.clearContextsList = function() { - return this.setContextsList([]); -}; - - +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.repeatedFields_ = [1]; @@ -17804,8 +23927,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByExecutionRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.toObject(opt_includeInstance, this); }; @@ -17814,14 +23937,14 @@ proto.ml_metadata.GetContextsByExecutionRequest.prototype.toObject = function(op * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByExecutionRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByExecutionRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.toObject = function(includeInstance, msg) { var f, obj = { - executionId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + parentContextsList: jspb.Message.toObjectList(msg.getParentContextsList(), + ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) }; if (includeInstance) { @@ -17835,23 +23958,23 @@ proto.ml_metadata.GetContextsByExecutionRequest.toObject = function(includeInsta /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByExecutionRequest} + * @return {!proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild} */ -proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByExecutionRequest; - return proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild; + return proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByExecutionRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByExecutionRequest} + * @return {!proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild} */ -proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -17859,13 +23982,9 @@ proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinaryFromReader = fu var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setExecutionId(value); - break; - case 2: - var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); - msg.setTransactionOptions(value); + var value = new ml_metadata_proto_metadata_store_pb.Context; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); + msg.addParentContexts(value); break; default: reader.skipField(); @@ -17880,9 +23999,9 @@ proto.ml_metadata.GetContextsByExecutionRequest.deserializeBinaryFromReader = fu * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByExecutionRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -17890,101 +24009,81 @@ proto.ml_metadata.GetContextsByExecutionRequest.prototype.serializeBinary = func /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByExecutionRequest} message + * @param {!proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByExecutionRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( + f = message.getParentContextsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( 1, - f - ); - } - f = message.getTransactionOptions(); - if (f != null) { - writer.writeMessage( - 2, f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter ); } }; /** - * optional int64 execution_id = 1; - * @return {number} - */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.getExecutionId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); -}; - - -/** - * @param {number} value - * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this + * repeated Context parent_contexts = 1; + * @return {!Array} */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.setExecutionId = function(value) { - return jspb.Message.setField(this, 1, value); +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.prototype.getParentContextsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; /** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this - */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.clearExecutionId = function() { - return jspb.Message.setField(this, 1, undefined); + * @param {!Array} value + * @return {!proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild} returns this +*/ +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.prototype.setParentContextsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * Returns whether this field is set. - * @return {boolean} + * @param {!proto.ml_metadata.Context=} opt_value + * @param {number=} opt_index + * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.hasExecutionId = function() { - return jspb.Message.getField(this, 1) != null; +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.prototype.addParentContexts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** - * optional TransactionOptions transaction_options = 2; - * @return {?proto.ml_metadata.TransactionOptions} + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild} returns this */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.getTransactionOptions = function() { - return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); -}; - - -/** - * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this -*/ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.prototype.clearParentContextsList = function() { + return this.setParentContextsList([]); }; /** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetContextsByExecutionRequest} returns this + * map contexts = 2; + * @param {boolean=} opt_noLazyCreate Do not create the map if + * empty, instead returning `undefined` + * @return {!jspb.Map} */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.clearTransactionOptions = function() { - return this.setTransactionOptions(undefined); +proto.ml_metadata.GetParentContextsByContextsResponse.prototype.getContextsMap = function(opt_noLazyCreate) { + return /** @type {!jspb.Map} */ ( + jspb.Message.getMapField(this, 2, opt_noLazyCreate, + proto.ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild)); }; /** - * Returns whether this field is set. - * @return {boolean} + * Clears values from the map. The map will be non-null. + * @return {!proto.ml_metadata.GetParentContextsByContextsResponse} returns this */ -proto.ml_metadata.GetContextsByExecutionRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 2) != null; -}; +proto.ml_metadata.GetParentContextsByContextsResponse.prototype.clearContextsMap = function() { + this.getContextsMap().clear(); + return this;}; @@ -17993,7 +24092,7 @@ proto.ml_metadata.GetContextsByExecutionRequest.prototype.hasTransactionOptions * @private {!Array} * @const */ -proto.ml_metadata.GetContextsByExecutionResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetChildrenContextsByContextsRequest.repeatedFields_ = [1]; @@ -18010,8 +24109,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetContextsByExecutionResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetContextsByExecutionResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetChildrenContextsByContextsRequest.toObject(opt_includeInstance, this); }; @@ -18020,14 +24119,14 @@ proto.ml_metadata.GetContextsByExecutionResponse.prototype.toObject = function(o * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetContextsByExecutionResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetChildrenContextsByContextsRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByExecutionResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetChildrenContextsByContextsRequest.toObject = function(includeInstance, msg) { var f, obj = { - contextsList: jspb.Message.toObjectList(msg.getContextsList(), - ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) + contextIdsList: (f = jspb.Message.getRepeatedField(msg, 1)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -18041,23 +24140,23 @@ proto.ml_metadata.GetContextsByExecutionResponse.toObject = function(includeInst /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetContextsByExecutionResponse} + * @return {!proto.ml_metadata.GetChildrenContextsByContextsRequest} */ -proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetChildrenContextsByContextsRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetContextsByExecutionResponse; - return proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetChildrenContextsByContextsRequest; + return proto.ml_metadata.GetChildrenContextsByContextsRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetContextsByExecutionResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetChildrenContextsByContextsRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetContextsByExecutionResponse} + * @return {!proto.ml_metadata.GetChildrenContextsByContextsRequest} */ -proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetChildrenContextsByContextsRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -18065,9 +24164,15 @@ proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinaryFromReader = f var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Context; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); - msg.addContexts(value); + var values = /** @type {!Array} */ (reader.isDelimited() ? reader.readPackedInt64() : [reader.readInt64()]); + for (var i = 0; i < values.length; i++) { + msg.addContextIds(values[i]); + } + break; + case 2: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); break; default: reader.skipField(); @@ -18082,9 +24187,9 @@ proto.ml_metadata.GetContextsByExecutionResponse.deserializeBinaryFromReader = f * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetContextsByExecutionResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetContextsByExecutionResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetChildrenContextsByContextsRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -18092,58 +24197,101 @@ proto.ml_metadata.GetContextsByExecutionResponse.prototype.serializeBinary = fun /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetContextsByExecutionResponse} message + * @param {!proto.ml_metadata.GetChildrenContextsByContextsRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetContextsByExecutionResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetChildrenContextsByContextsRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextsList(); + f = message.getContextIdsList(); if (f.length > 0) { - writer.writeRepeatedMessage( + writer.writePackedInt64( 1, + f + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 2, f, - ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } }; /** - * repeated Context contexts = 1; - * @return {!Array} + * repeated int64 context_ids = 1; + * @return {!Array} */ -proto.ml_metadata.GetContextsByExecutionResponse.prototype.getContextsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.getContextIdsList = function() { + return /** @type {!Array} */ (jspb.Message.getRepeatedField(this, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetContextsByExecutionResponse} returns this -*/ -proto.ml_metadata.GetContextsByExecutionResponse.prototype.setContextsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); + * @param {!Array} value + * @return {!proto.ml_metadata.GetChildrenContextsByContextsRequest} returns this + */ +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.setContextIdsList = function(value) { + return jspb.Message.setField(this, 1, value || []); }; /** - * @param {!proto.ml_metadata.Context=} opt_value + * @param {number} value * @param {number=} opt_index - * @return {!proto.ml_metadata.Context} + * @return {!proto.ml_metadata.GetChildrenContextsByContextsRequest} returns this */ -proto.ml_metadata.GetContextsByExecutionResponse.prototype.addContexts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.addContextIds = function(value, opt_index) { + return jspb.Message.addToRepeatedField(this, 1, value, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.ml_metadata.GetChildrenContextsByContextsRequest} returns this + */ +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.clearContextIdsList = function() { + return this.setContextIdsList([]); +}; + + +/** + * optional TransactionOptions transaction_options = 2; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetChildrenContextsByContextsRequest} returns this +*/ +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetChildrenContextsByContextsRequest} returns this + */ +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); }; /** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetContextsByExecutionResponse} returns this + * Returns whether this field is set. + * @return {boolean} */ -proto.ml_metadata.GetContextsByExecutionResponse.prototype.clearContextsList = function() { - return this.setContextsList([]); +proto.ml_metadata.GetChildrenContextsByContextsRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -18163,8 +24311,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetParentContextsByContextRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetChildrenContextsByContextsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetChildrenContextsByContextsResponse.toObject(opt_includeInstance, this); }; @@ -18173,14 +24321,13 @@ proto.ml_metadata.GetParentContextsByContextRequest.prototype.toObject = functio * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetParentContextsByContextRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetChildrenContextsByContextsResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetParentContextsByContextRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.toObject = function(includeInstance, msg) { var f, obj = { - contextId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + contextsMap: (f = msg.getContextsMap()) ? f.toObject(includeInstance, proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.toObject) : [] }; if (includeInstance) { @@ -18194,37 +24341,34 @@ proto.ml_metadata.GetParentContextsByContextRequest.toObject = function(includeI /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetParentContextsByContextRequest} + * @return {!proto.ml_metadata.GetChildrenContextsByContextsResponse} */ -proto.ml_metadata.GetParentContextsByContextRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetParentContextsByContextRequest; - return proto.ml_metadata.GetParentContextsByContextRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetChildrenContextsByContextsResponse; + return proto.ml_metadata.GetChildrenContextsByContextsResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetParentContextsByContextRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetChildrenContextsByContextsResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetParentContextsByContextRequest} + * @return {!proto.ml_metadata.GetChildrenContextsByContextsResponse} */ -proto.ml_metadata.GetParentContextsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; } var field = reader.getFieldNumber(); switch (field) { - case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setContextId(value); - break; case 2: - var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); - msg.setTransactionOptions(value); + var value = msg.getContextsMap(); + reader.readMessage(value, function(message, reader) { + jspb.Map.deserializeBinary(message, reader, jspb.BinaryReader.prototype.readInt64, jspb.BinaryReader.prototype.readMessage, proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.deserializeBinaryFromReader, 0, new proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent()); + }); break; default: reader.skipField(); @@ -18239,9 +24383,9 @@ proto.ml_metadata.GetParentContextsByContextRequest.deserializeBinaryFromReader * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetChildrenContextsByContextsResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetParentContextsByContextRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetChildrenContextsByContextsResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -18249,110 +24393,26 @@ proto.ml_metadata.GetParentContextsByContextRequest.prototype.serializeBinary = /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetParentContextsByContextRequest} message + * @param {!proto.ml_metadata.GetChildrenContextsByContextsResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetParentContextsByContextRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( - 1, - f - ); - } - f = message.getTransactionOptions(); - if (f != null) { - writer.writeMessage( - 2, - f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter - ); + f = message.getContextsMap(true); + if (f && f.getLength() > 0) { + f.serializeBinary(2, writer, jspb.BinaryWriter.prototype.writeInt64, jspb.BinaryWriter.prototype.writeMessage, proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.serializeBinaryToWriter); } }; -/** - * optional int64 context_id = 1; - * @return {number} - */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.getContextId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); -}; - - -/** - * @param {number} value - * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this - */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.setContextId = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this - */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.clearContextId = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.hasContextId = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * optional TransactionOptions transaction_options = 2; - * @return {?proto.ml_metadata.TransactionOptions} - */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.getTransactionOptions = function() { - return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); -}; - - -/** - * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this -*/ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); -}; - - -/** - * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetParentContextsByContextRequest} returns this - */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.clearTransactionOptions = function() { - return this.setTransactionOptions(undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetParentContextsByContextRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 2) != null; -}; - - /** * List of repeated fields within this message type. * @private {!Array} * @const */ -proto.ml_metadata.GetParentContextsByContextResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.repeatedFields_ = [1]; @@ -18369,8 +24429,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetParentContextsByContextResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetParentContextsByContextResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.toObject(opt_includeInstance, this); }; @@ -18379,13 +24439,13 @@ proto.ml_metadata.GetParentContextsByContextResponse.prototype.toObject = functi * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetParentContextsByContextResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetParentContextsByContextResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.toObject = function(includeInstance, msg) { var f, obj = { - contextsList: jspb.Message.toObjectList(msg.getContextsList(), + childrenContextsList: jspb.Message.toObjectList(msg.getChildrenContextsList(), ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) }; @@ -18400,23 +24460,23 @@ proto.ml_metadata.GetParentContextsByContextResponse.toObject = function(include /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetParentContextsByContextResponse} + * @return {!proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent} */ -proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetParentContextsByContextResponse; - return proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent; + return proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetParentContextsByContextResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetParentContextsByContextResponse} + * @return {!proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent} */ -proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -18426,7 +24486,7 @@ proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinaryFromReader case 1: var value = new ml_metadata_proto_metadata_store_pb.Context; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); - msg.addContexts(value); + msg.addChildrenContexts(value); break; default: reader.skipField(); @@ -18441,9 +24501,9 @@ proto.ml_metadata.GetParentContextsByContextResponse.deserializeBinaryFromReader * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetParentContextsByContextResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetParentContextsByContextResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -18451,13 +24511,13 @@ proto.ml_metadata.GetParentContextsByContextResponse.prototype.serializeBinary = /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetParentContextsByContextResponse} message + * @param {!proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetParentContextsByContextResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextsList(); + f = message.getChildrenContextsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, @@ -18469,10 +24529,10 @@ proto.ml_metadata.GetParentContextsByContextResponse.serializeBinaryToWriter = f /** - * repeated Context contexts = 1; + * repeated Context children_contexts = 1; * @return {!Array} */ -proto.ml_metadata.GetParentContextsByContextResponse.prototype.getContextsList = function() { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.prototype.getChildrenContextsList = function() { return /** @type{!Array} */ ( jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); }; @@ -18480,9 +24540,9 @@ proto.ml_metadata.GetParentContextsByContextResponse.prototype.getContextsList = /** * @param {!Array} value - * @return {!proto.ml_metadata.GetParentContextsByContextResponse} returns this + * @return {!proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent} returns this */ -proto.ml_metadata.GetParentContextsByContextResponse.prototype.setContextsList = function(value) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.prototype.setChildrenContextsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; @@ -18492,20 +24552,42 @@ proto.ml_metadata.GetParentContextsByContextResponse.prototype.setContextsList = * @param {number=} opt_index * @return {!proto.ml_metadata.Context} */ -proto.ml_metadata.GetParentContextsByContextResponse.prototype.addContexts = function(opt_value, opt_index) { +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.prototype.addChildrenContexts = function(opt_value, opt_index) { return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetParentContextsByContextResponse} returns this + * @return {!proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent} returns this */ -proto.ml_metadata.GetParentContextsByContextResponse.prototype.clearContextsList = function() { - return this.setContextsList([]); +proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.prototype.clearChildrenContextsList = function() { + return this.setChildrenContextsList([]); +}; + + +/** + * map contexts = 2; + * @param {boolean=} opt_noLazyCreate Do not create the map if + * empty, instead returning `undefined` + * @return {!jspb.Map} + */ +proto.ml_metadata.GetChildrenContextsByContextsResponse.prototype.getContextsMap = function(opt_noLazyCreate) { + return /** @type {!jspb.Map} */ ( + jspb.Message.getMapField(this, 2, opt_noLazyCreate, + proto.ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent)); }; +/** + * Clears values from the map. The map will be non-null. + * @return {!proto.ml_metadata.GetChildrenContextsByContextsResponse} returns this + */ +proto.ml_metadata.GetChildrenContextsByContextsResponse.prototype.clearContextsMap = function() { + this.getContextsMap().clear(); + return this;}; + + @@ -18522,8 +24604,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetChildrenContextsByContextRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByContextRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByContextRequest.toObject(opt_includeInstance, this); }; @@ -18532,13 +24614,14 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.toObject = funct * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetChildrenContextsByContextRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByContextRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetChildrenContextsByContextRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByContextRequest.toObject = function(includeInstance, msg) { var f, obj = { contextId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, + options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -18553,23 +24636,23 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.toObject = function(includ /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByContextRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetChildrenContextsByContextRequest; - return proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByContextRequest; + return proto.ml_metadata.GetArtifactsByContextRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetChildrenContextsByContextRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByContextRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -18581,6 +24664,11 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinaryFromReade msg.setContextId(value); break; case 2: + var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); + msg.setOptions(value); + break; + case 3: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -18598,9 +24686,9 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.deserializeBinaryFromReade * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByContextRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetChildrenContextsByContextRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByContextRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -18608,11 +24696,11 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.serializeBinary /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetChildrenContextsByContextRequest} message + * @param {!proto.ml_metadata.GetArtifactsByContextRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetChildrenContextsByContextRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByContextRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; f = /** @type {number} */ (jspb.Message.getField(message, 1)); if (f != null) { @@ -18621,11 +24709,19 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.serializeBinaryToWriter = f ); } - f = message.getTransactionOptions(); + f = message.getOptions(); if (f != null) { writer.writeMessage( 2, f, + ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter + ); + } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 3, + f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); } @@ -18636,25 +24732,25 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.serializeBinaryToWriter = * optional int64 context_id = 1; * @return {number} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.getContextId = function() { +proto.ml_metadata.GetArtifactsByContextRequest.prototype.getContextId = function() { return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** * @param {number} value - * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.setContextId = function(value) { +proto.ml_metadata.GetArtifactsByContextRequest.prototype.setContextId = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.clearContextId = function() { +proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearContextId = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -18663,35 +24759,72 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.clearContextId = * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.hasContextId = function() { +proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasContextId = function() { return jspb.Message.getField(this, 1) != null; }; /** - * optional TransactionOptions transaction_options = 2; + * optional ListOperationOptions options = 2; + * @return {?proto.ml_metadata.ListOperationOptions} + */ +proto.ml_metadata.GetArtifactsByContextRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 2)); +}; + + +/** + * @param {?proto.ml_metadata.ListOperationOptions|undefined} value + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this +*/ +proto.ml_metadata.GetArtifactsByContextRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this + */ +proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearOptions = function() { + return this.setOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional TransactionOptions transaction_options = 3; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetArtifactsByContextRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.ml_metadata.GetArtifactsByContextRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetChildrenContextsByContextRequest} returns this + * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -18700,8 +24833,8 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.clearTransaction * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; }; @@ -18711,7 +24844,7 @@ proto.ml_metadata.GetChildrenContextsByContextRequest.prototype.hasTransactionOp * @private {!Array} * @const */ -proto.ml_metadata.GetChildrenContextsByContextResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetArtifactsByContextResponse.repeatedFields_ = [1]; @@ -18728,8 +24861,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetChildrenContextsByContextResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetArtifactsByContextResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetArtifactsByContextResponse.toObject(opt_includeInstance, this); }; @@ -18738,14 +24871,15 @@ proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.toObject = func * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetChildrenContextsByContextResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetArtifactsByContextResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetChildrenContextsByContextResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetArtifactsByContextResponse.toObject = function(includeInstance, msg) { var f, obj = { - contextsList: jspb.Message.toObjectList(msg.getContextsList(), - ml_metadata_proto_metadata_store_pb.Context.toObject, includeInstance) + artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), + ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f }; if (includeInstance) { @@ -18759,23 +24893,23 @@ proto.ml_metadata.GetChildrenContextsByContextResponse.toObject = function(inclu /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} + * @return {!proto.ml_metadata.GetArtifactsByContextResponse} */ -proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetChildrenContextsByContextResponse; - return proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetArtifactsByContextResponse; + return proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetChildrenContextsByContextResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetArtifactsByContextResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} + * @return {!proto.ml_metadata.GetArtifactsByContextResponse} */ -proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -18783,9 +24917,13 @@ proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinaryFromRead var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Context; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Context.deserializeBinaryFromReader); - msg.addContexts(value); + var value = new ml_metadata_proto_metadata_store_pb.Artifact; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); + msg.addArtifacts(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setNextPageToken(value); break; default: reader.skipField(); @@ -18800,9 +24938,9 @@ proto.ml_metadata.GetChildrenContextsByContextResponse.deserializeBinaryFromRead * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetArtifactsByContextResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetChildrenContextsByContextResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetArtifactsByContextResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -18810,58 +24948,101 @@ proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.serializeBinary /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetChildrenContextsByContextResponse} message + * @param {!proto.ml_metadata.GetArtifactsByContextResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetChildrenContextsByContextResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetArtifactsByContextResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getContextsList(); + f = message.getArtifactsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Context.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ); + } + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( + 2, + f ); } }; /** - * repeated Context contexts = 1; - * @return {!Array} + * repeated Artifact artifacts = 1; + * @return {!Array} */ -proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.getContextsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Context, 1)); +proto.ml_metadata.GetArtifactsByContextResponse.prototype.getArtifactsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this */ -proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.setContextsList = function(value) { +proto.ml_metadata.GetArtifactsByContextResponse.prototype.setArtifactsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Context=} opt_value + * @param {!proto.ml_metadata.Artifact=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Context} + * @return {!proto.ml_metadata.Artifact} */ -proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.addContexts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Context, opt_index); +proto.ml_metadata.GetArtifactsByContextResponse.prototype.addArtifacts = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetChildrenContextsByContextResponse} returns this + * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this */ -proto.ml_metadata.GetChildrenContextsByContextResponse.prototype.clearContextsList = function() { - return this.setContextsList([]); +proto.ml_metadata.GetArtifactsByContextResponse.prototype.clearArtifactsList = function() { + return this.setArtifactsList([]); +}; + + +/** + * optional string next_page_token = 2; + * @return {string} + */ +proto.ml_metadata.GetArtifactsByContextResponse.prototype.getNextPageToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this + */ +proto.ml_metadata.GetArtifactsByContextResponse.prototype.setNextPageToken = function(value) { + return jspb.Message.setField(this, 2, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this + */ +proto.ml_metadata.GetArtifactsByContextResponse.prototype.clearNextPageToken = function() { + return jspb.Message.setField(this, 2, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetArtifactsByContextResponse.prototype.hasNextPageToken = function() { + return jspb.Message.getField(this, 2) != null; }; @@ -18881,8 +25062,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByContextRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByContextRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByContextRequest.toObject(opt_includeInstance, this); }; @@ -18891,11 +25072,11 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.toObject = function(opt * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByContextRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByContextRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByContextRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByContextRequest.toObject = function(includeInstance, msg) { var f, obj = { contextId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), @@ -18913,23 +25094,23 @@ proto.ml_metadata.GetArtifactsByContextRequest.toObject = function(includeInstan /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} */ -proto.ml_metadata.GetArtifactsByContextRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByContextRequest; - return proto.ml_metadata.GetArtifactsByContextRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByContextRequest; + return proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByContextRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByContextRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} */ -proto.ml_metadata.GetArtifactsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -18963,9 +25144,9 @@ proto.ml_metadata.GetArtifactsByContextRequest.deserializeBinaryFromReader = fun * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByContextRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByContextRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -18973,11 +25154,11 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.serializeBinary = funct /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByContextRequest} message + * @param {!proto.ml_metadata.GetExecutionsByContextRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByContextRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByContextRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; f = /** @type {number} */ (jspb.Message.getField(message, 1)); if (f != null) { @@ -19009,25 +25190,25 @@ proto.ml_metadata.GetArtifactsByContextRequest.serializeBinaryToWriter = functio * optional int64 context_id = 1; * @return {number} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.getContextId = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.getContextId = function() { return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); }; /** * @param {number} value - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.setContextId = function(value) { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.setContextId = function(value) { return jspb.Message.setField(this, 1, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearContextId = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearContextId = function() { return jspb.Message.setField(this, 1, undefined); }; @@ -19036,7 +25217,7 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearContextId = functi * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasContextId = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.hasContextId = function() { return jspb.Message.getField(this, 1) != null; }; @@ -19045,7 +25226,7 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasContextId = function * optional ListOperationOptions options = 2; * @return {?proto.ml_metadata.ListOperationOptions} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.getOptions = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.getOptions = function() { return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 2)); }; @@ -19053,18 +25234,18 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.getOptions = function() /** * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.setOptions = function(value) { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.setOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearOptions = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearOptions = function() { return this.setOptions(undefined); }; @@ -19073,7 +25254,7 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearOptions = function * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasOptions = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.hasOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -19082,7 +25263,7 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasOptions = function() * optional TransactionOptions transaction_options = 3; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); }; @@ -19090,18 +25271,18 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.getTransactionOptions = /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 3, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByContextRequest} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -19110,7 +25291,7 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.clearTransactionOptions * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetExecutionsByContextRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 3) != null; }; @@ -19121,7 +25302,7 @@ proto.ml_metadata.GetArtifactsByContextRequest.prototype.hasTransactionOptions = * @private {!Array} * @const */ -proto.ml_metadata.GetArtifactsByContextResponse.repeatedFields_ = [1]; +proto.ml_metadata.GetExecutionsByContextResponse.repeatedFields_ = [1]; @@ -19138,8 +25319,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetArtifactsByContextResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetExecutionsByContextResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetExecutionsByContextResponse.toObject(opt_includeInstance, this); }; @@ -19148,15 +25329,16 @@ proto.ml_metadata.GetArtifactsByContextResponse.prototype.toObject = function(op * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetArtifactsByContextResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetExecutionsByContextResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByContextResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetExecutionsByContextResponse.toObject = function(includeInstance, msg) { var f, obj = { - artifactsList: jspb.Message.toObjectList(msg.getArtifactsList(), - ml_metadata_proto_metadata_store_pb.Artifact.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f + executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), + ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), + nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, + transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; if (includeInstance) { @@ -19170,23 +25352,23 @@ proto.ml_metadata.GetArtifactsByContextResponse.toObject = function(includeInsta /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetArtifactsByContextResponse} + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} */ -proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetArtifactsByContextResponse; - return proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetExecutionsByContextResponse; + return proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetArtifactsByContextResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetExecutionsByContextResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetArtifactsByContextResponse} + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} */ -proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -19194,14 +25376,19 @@ proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinaryFromReader = fu var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Artifact; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Artifact.deserializeBinaryFromReader); - msg.addArtifacts(value); + var value = new ml_metadata_proto_metadata_store_pb.Execution; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); + msg.addExecutions(value); break; case 2: var value = /** @type {string} */ (reader.readString()); msg.setNextPageToken(value); break; + case 3: + var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); + msg.setTransactionOptions(value); + break; default: reader.skipField(); break; @@ -19215,9 +25402,9 @@ proto.ml_metadata.GetArtifactsByContextResponse.deserializeBinaryFromReader = fu * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetExecutionsByContextResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetArtifactsByContextResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetExecutionsByContextResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -19225,18 +25412,18 @@ proto.ml_metadata.GetArtifactsByContextResponse.prototype.serializeBinary = func /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetArtifactsByContextResponse} message + * @param {!proto.ml_metadata.GetExecutionsByContextResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetArtifactsByContextResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetExecutionsByContextResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getArtifactsList(); + f = message.getExecutionsList(); if (f.length > 0) { writer.writeRepeatedMessage( 1, f, - ml_metadata_proto_metadata_store_pb.Artifact.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter ); } f = /** @type {string} */ (jspb.Message.getField(message, 2)); @@ -19246,44 +25433,52 @@ proto.ml_metadata.GetArtifactsByContextResponse.serializeBinaryToWriter = functi f ); } + f = message.getTransactionOptions(); + if (f != null) { + writer.writeMessage( + 3, + f, + ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ); + } }; /** - * repeated Artifact artifacts = 1; - * @return {!Array} + * repeated Execution executions = 1; + * @return {!Array} */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.getArtifactsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Artifact, 1)); +proto.ml_metadata.GetExecutionsByContextResponse.prototype.getExecutionsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); }; /** - * @param {!Array} value - * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this + * @param {!Array} value + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.setArtifactsList = function(value) { +proto.ml_metadata.GetExecutionsByContextResponse.prototype.setExecutionsList = function(value) { return jspb.Message.setRepeatedWrapperField(this, 1, value); }; /** - * @param {!proto.ml_metadata.Artifact=} opt_value + * @param {!proto.ml_metadata.Execution=} opt_value * @param {number=} opt_index - * @return {!proto.ml_metadata.Artifact} + * @return {!proto.ml_metadata.Execution} */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.addArtifacts = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Artifact, opt_index); +proto.ml_metadata.GetExecutionsByContextResponse.prototype.addExecutions = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); }; /** * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.clearArtifactsList = function() { - return this.setArtifactsList([]); +proto.ml_metadata.GetExecutionsByContextResponse.prototype.clearExecutionsList = function() { + return this.setExecutionsList([]); }; @@ -19291,25 +25486,25 @@ proto.ml_metadata.GetArtifactsByContextResponse.prototype.clearArtifactsList = f * optional string next_page_token = 2; * @return {string} */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.getNextPageToken = function() { +proto.ml_metadata.GetExecutionsByContextResponse.prototype.getNextPageToken = function() { return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; /** * @param {string} value - * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.setNextPageToken = function(value) { +proto.ml_metadata.GetExecutionsByContextResponse.prototype.setNextPageToken = function(value) { return jspb.Message.setField(this, 2, value); }; /** * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetArtifactsByContextResponse} returns this + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.clearNextPageToken = function() { +proto.ml_metadata.GetExecutionsByContextResponse.prototype.clearNextPageToken = function() { return jspb.Message.setField(this, 2, undefined); }; @@ -19318,11 +25513,48 @@ proto.ml_metadata.GetArtifactsByContextResponse.prototype.clearNextPageToken = f * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetArtifactsByContextResponse.prototype.hasNextPageToken = function() { +proto.ml_metadata.GetExecutionsByContextResponse.prototype.hasNextPageToken = function() { return jspb.Message.getField(this, 2) != null; }; +/** + * optional TransactionOptions transaction_options = 3; + * @return {?proto.ml_metadata.TransactionOptions} + */ +proto.ml_metadata.GetExecutionsByContextResponse.prototype.getTransactionOptions = function() { + return /** @type{?proto.ml_metadata.TransactionOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); +}; + + +/** + * @param {?proto.ml_metadata.TransactionOptions|undefined} value + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this +*/ +proto.ml_metadata.GetExecutionsByContextResponse.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this + */ +proto.ml_metadata.GetExecutionsByContextResponse.prototype.clearTransactionOptions = function() { + return this.setTransactionOptions(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetExecutionsByContextResponse.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 3) != null; +}; + + @@ -19339,8 +25571,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsByContextRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetLineageGraphRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetLineageGraphRequest.toObject(opt_includeInstance, this); }; @@ -19349,14 +25581,13 @@ proto.ml_metadata.GetExecutionsByContextRequest.prototype.toObject = function(op * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsByContextRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetLineageGraphRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByContextRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetLineageGraphRequest.toObject = function(includeInstance, msg) { var f, obj = { - contextId: (f = jspb.Message.getField(msg, 1)) == null ? undefined : f, - options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.ListOperationOptions.toObject(includeInstance, f), + options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions.toObject(includeInstance, f), transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -19371,23 +25602,23 @@ proto.ml_metadata.GetExecutionsByContextRequest.toObject = function(includeInsta /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} + * @return {!proto.ml_metadata.GetLineageGraphRequest} */ -proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetLineageGraphRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsByContextRequest; - return proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetLineageGraphRequest; + return proto.ml_metadata.GetLineageGraphRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsByContextRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetLineageGraphRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} + * @return {!proto.ml_metadata.GetLineageGraphRequest} */ -proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetLineageGraphRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -19395,15 +25626,11 @@ proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinaryFromReader = fu var field = reader.getFieldNumber(); switch (field) { case 1: - var value = /** @type {number} */ (reader.readInt64()); - msg.setContextId(value); - break; - case 2: - var value = new ml_metadata_proto_metadata_store_pb.ListOperationOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.ListOperationOptions.deserializeBinaryFromReader); + var value = new ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions.deserializeBinaryFromReader); msg.setOptions(value); break; - case 3: + case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); msg.setTransactionOptions(value); @@ -19421,9 +25648,9 @@ proto.ml_metadata.GetExecutionsByContextRequest.deserializeBinaryFromReader = fu * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetLineageGraphRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsByContextRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetLineageGraphRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -19431,31 +25658,24 @@ proto.ml_metadata.GetExecutionsByContextRequest.prototype.serializeBinary = func /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsByContextRequest} message + * @param {!proto.ml_metadata.GetLineageGraphRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByContextRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetLineageGraphRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = /** @type {number} */ (jspb.Message.getField(message, 1)); - if (f != null) { - writer.writeInt64( - 1, - f - ); - } f = message.getOptions(); if (f != null) { writer.writeMessage( - 2, + 1, f, - ml_metadata_proto_metadata_store_pb.ListOperationOptions.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions.serializeBinaryToWriter ); } f = message.getTransactionOptions(); if (f != null) { writer.writeMessage( - 3, + 2, f, ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter ); @@ -19464,65 +25684,29 @@ proto.ml_metadata.GetExecutionsByContextRequest.serializeBinaryToWriter = functi /** - * optional int64 context_id = 1; - * @return {number} - */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.getContextId = function() { - return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); -}; - - -/** - * @param {number} value - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this - */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.setContextId = function(value) { - return jspb.Message.setField(this, 1, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this - */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearContextId = function() { - return jspb.Message.setField(this, 1, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.hasContextId = function() { - return jspb.Message.getField(this, 1) != null; -}; - - -/** - * optional ListOperationOptions options = 2; - * @return {?proto.ml_metadata.ListOperationOptions} + * optional LineageGraphQueryOptions options = 1; + * @return {?proto.ml_metadata.LineageGraphQueryOptions} */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.ListOperationOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.ListOperationOptions, 2)); +proto.ml_metadata.GetLineageGraphRequest.prototype.getOptions = function() { + return /** @type{?proto.ml_metadata.LineageGraphQueryOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions, 1)); }; /** - * @param {?proto.ml_metadata.ListOperationOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this + * @param {?proto.ml_metadata.LineageGraphQueryOptions|undefined} value + * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.setOptions = function(value) { - return jspb.Message.setWrapperField(this, 2, value); +proto.ml_metadata.GetLineageGraphRequest.prototype.setOptions = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this + * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearOptions = function() { +proto.ml_metadata.GetLineageGraphRequest.prototype.clearOptions = function() { return this.setOptions(undefined); }; @@ -19531,35 +25715,35 @@ proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearOptions = functio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.hasOptions = function() { - return jspb.Message.getField(this, 2) != null; +proto.ml_metadata.GetLineageGraphRequest.prototype.hasOptions = function() { + return jspb.Message.getField(this, 1) != null; }; /** - * optional TransactionOptions transaction_options = 3; + * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetLineageGraphRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this + * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); +proto.ml_metadata.GetLineageGraphRequest.prototype.setTransactionOptions = function(value) { + return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByContextRequest} returns this + * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetLineageGraphRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -19568,19 +25752,12 @@ proto.ml_metadata.GetExecutionsByContextRequest.prototype.clearTransactionOption * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionsByContextRequest.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetLineageGraphRequest.prototype.hasTransactionOptions = function() { + return jspb.Message.getField(this, 2) != null; }; -/** - * List of repeated fields within this message type. - * @private {!Array} - * @const - */ -proto.ml_metadata.GetExecutionsByContextResponse.repeatedFields_ = [1]; - if (jspb.Message.GENERATE_TO_OBJECT) { @@ -19596,8 +25773,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetExecutionsByContextResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetLineageGraphResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetLineageGraphResponse.toObject(opt_includeInstance, this); }; @@ -19606,16 +25783,13 @@ proto.ml_metadata.GetExecutionsByContextResponse.prototype.toObject = function(o * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetExecutionsByContextResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetLineageGraphResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByContextResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetLineageGraphResponse.toObject = function(includeInstance, msg) { var f, obj = { - executionsList: jspb.Message.toObjectList(msg.getExecutionsList(), - ml_metadata_proto_metadata_store_pb.Execution.toObject, includeInstance), - nextPageToken: (f = jspb.Message.getField(msg, 2)) == null ? undefined : f, - transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) + subgraph: (f = msg.getSubgraph()) && ml_metadata_proto_metadata_store_pb.LineageGraph.toObject(includeInstance, f) }; if (includeInstance) { @@ -19629,23 +25803,23 @@ proto.ml_metadata.GetExecutionsByContextResponse.toObject = function(includeInst /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} + * @return {!proto.ml_metadata.GetLineageGraphResponse} */ -proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetLineageGraphResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetExecutionsByContextResponse; - return proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetLineageGraphResponse; + return proto.ml_metadata.GetLineageGraphResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetExecutionsByContextResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetLineageGraphResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} + * @return {!proto.ml_metadata.GetLineageGraphResponse} */ -proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetLineageGraphResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -19653,18 +25827,9 @@ proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinaryFromReader = f var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.Execution; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.Execution.deserializeBinaryFromReader); - msg.addExecutions(value); - break; - case 2: - var value = /** @type {string} */ (reader.readString()); - msg.setNextPageToken(value); - break; - case 3: - var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.TransactionOptions.deserializeBinaryFromReader); - msg.setTransactionOptions(value); + var value = new ml_metadata_proto_metadata_store_pb.LineageGraph; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.LineageGraph.deserializeBinaryFromReader); + msg.setSubgraph(value); break; default: reader.skipField(); @@ -19679,9 +25844,9 @@ proto.ml_metadata.GetExecutionsByContextResponse.deserializeBinaryFromReader = f * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetLineageGraphResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetExecutionsByContextResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetLineageGraphResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -19689,137 +25854,48 @@ proto.ml_metadata.GetExecutionsByContextResponse.prototype.serializeBinary = fun /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetExecutionsByContextResponse} message + * @param {!proto.ml_metadata.GetLineageGraphResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetExecutionsByContextResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetLineageGraphResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getExecutionsList(); - if (f.length > 0) { - writer.writeRepeatedMessage( - 1, - f, - ml_metadata_proto_metadata_store_pb.Execution.serializeBinaryToWriter - ); - } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( - 2, - f - ); - } - f = message.getTransactionOptions(); + f = message.getSubgraph(); if (f != null) { writer.writeMessage( - 3, + 1, f, - ml_metadata_proto_metadata_store_pb.TransactionOptions.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.LineageGraph.serializeBinaryToWriter ); } }; /** - * repeated Execution executions = 1; - * @return {!Array} - */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.getExecutionsList = function() { - return /** @type{!Array} */ ( - jspb.Message.getRepeatedWrapperField(this, ml_metadata_proto_metadata_store_pb.Execution, 1)); -}; - - -/** - * @param {!Array} value - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this -*/ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.setExecutionsList = function(value) { - return jspb.Message.setRepeatedWrapperField(this, 1, value); -}; - - -/** - * @param {!proto.ml_metadata.Execution=} opt_value - * @param {number=} opt_index - * @return {!proto.ml_metadata.Execution} - */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.addExecutions = function(opt_value, opt_index) { - return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.ml_metadata.Execution, opt_index); -}; - - -/** - * Clears the list making it empty but non-null. - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this - */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.clearExecutionsList = function() { - return this.setExecutionsList([]); -}; - - -/** - * optional string next_page_token = 2; - * @return {string} - */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.getNextPageToken = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); -}; - - -/** - * @param {string} value - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this - */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.setNextPageToken = function(value) { - return jspb.Message.setField(this, 2, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this - */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.clearNextPageToken = function() { - return jspb.Message.setField(this, 2, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.hasNextPageToken = function() { - return jspb.Message.getField(this, 2) != null; -}; - - -/** - * optional TransactionOptions transaction_options = 3; - * @return {?proto.ml_metadata.TransactionOptions} + * optional LineageGraph subgraph = 1; + * @return {?proto.ml_metadata.LineageGraph} */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.getTransactionOptions = function() { - return /** @type{?proto.ml_metadata.TransactionOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 3)); +proto.ml_metadata.GetLineageGraphResponse.prototype.getSubgraph = function() { + return /** @type{?proto.ml_metadata.LineageGraph} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.LineageGraph, 1)); }; /** - * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this + * @param {?proto.ml_metadata.LineageGraph|undefined} value + * @return {!proto.ml_metadata.GetLineageGraphResponse} returns this */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.setTransactionOptions = function(value) { - return jspb.Message.setWrapperField(this, 3, value); +proto.ml_metadata.GetLineageGraphResponse.prototype.setSubgraph = function(value) { + return jspb.Message.setWrapperField(this, 1, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetExecutionsByContextResponse} returns this + * @return {!proto.ml_metadata.GetLineageGraphResponse} returns this */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.clearTransactionOptions = function() { - return this.setTransactionOptions(undefined); +proto.ml_metadata.GetLineageGraphResponse.prototype.clearSubgraph = function() { + return this.setSubgraph(undefined); }; @@ -19827,8 +25903,8 @@ proto.ml_metadata.GetExecutionsByContextResponse.prototype.clearTransactionOptio * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetExecutionsByContextResponse.prototype.hasTransactionOptions = function() { - return jspb.Message.getField(this, 3) != null; +proto.ml_metadata.GetLineageGraphResponse.prototype.hasSubgraph = function() { + return jspb.Message.getField(this, 1) != null; }; @@ -19848,8 +25924,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetLineageGraphRequest.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetLineageGraphRequest.toObject(opt_includeInstance, this); +proto.ml_metadata.GetLineageSubgraphRequest.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetLineageSubgraphRequest.toObject(opt_includeInstance, this); }; @@ -19858,13 +25934,14 @@ proto.ml_metadata.GetLineageGraphRequest.prototype.toObject = function(opt_inclu * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetLineageGraphRequest} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetLineageSubgraphRequest} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetLineageGraphRequest.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetLineageSubgraphRequest.toObject = function(includeInstance, msg) { var f, obj = { - options: (f = msg.getOptions()) && ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions.toObject(includeInstance, f), + lineageSubgraphQueryOptions: (f = msg.getLineageSubgraphQueryOptions()) && ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions.toObject(includeInstance, f), + readMask: (f = msg.getReadMask()) && google_protobuf_field_mask_pb.FieldMask.toObject(includeInstance, f), transactionOptions: (f = msg.getTransactionOptions()) && ml_metadata_proto_metadata_store_pb.TransactionOptions.toObject(includeInstance, f) }; @@ -19879,23 +25956,23 @@ proto.ml_metadata.GetLineageGraphRequest.toObject = function(includeInstance, ms /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetLineageGraphRequest} + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} */ -proto.ml_metadata.GetLineageGraphRequest.deserializeBinary = function(bytes) { +proto.ml_metadata.GetLineageSubgraphRequest.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetLineageGraphRequest; - return proto.ml_metadata.GetLineageGraphRequest.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetLineageSubgraphRequest; + return proto.ml_metadata.GetLineageSubgraphRequest.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetLineageGraphRequest} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetLineageSubgraphRequest} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetLineageGraphRequest} + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} */ -proto.ml_metadata.GetLineageGraphRequest.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetLineageSubgraphRequest.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -19903,9 +25980,14 @@ proto.ml_metadata.GetLineageGraphRequest.deserializeBinaryFromReader = function( var field = reader.getFieldNumber(); switch (field) { case 1: - var value = new ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions; - reader.readMessage(value,ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions.deserializeBinaryFromReader); - msg.setOptions(value); + var value = new ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions; + reader.readMessage(value,ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions.deserializeBinaryFromReader); + msg.setLineageSubgraphQueryOptions(value); + break; + case 3: + var value = new google_protobuf_field_mask_pb.FieldMask; + reader.readMessage(value,google_protobuf_field_mask_pb.FieldMask.deserializeBinaryFromReader); + msg.setReadMask(value); break; case 2: var value = new ml_metadata_proto_metadata_store_pb.TransactionOptions; @@ -19925,9 +26007,9 @@ proto.ml_metadata.GetLineageGraphRequest.deserializeBinaryFromReader = function( * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetLineageGraphRequest.prototype.serializeBinary = function() { +proto.ml_metadata.GetLineageSubgraphRequest.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetLineageGraphRequest.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetLineageSubgraphRequest.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -19935,18 +26017,26 @@ proto.ml_metadata.GetLineageGraphRequest.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetLineageGraphRequest} message + * @param {!proto.ml_metadata.GetLineageSubgraphRequest} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetLineageGraphRequest.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetLineageSubgraphRequest.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getOptions(); + f = message.getLineageSubgraphQueryOptions(); if (f != null) { writer.writeMessage( 1, f, - ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions.serializeBinaryToWriter + ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions.serializeBinaryToWriter + ); + } + f = message.getReadMask(); + if (f != null) { + writer.writeMessage( + 3, + f, + google_protobuf_field_mask_pb.FieldMask.serializeBinaryToWriter ); } f = message.getTransactionOptions(); @@ -19961,30 +26051,30 @@ proto.ml_metadata.GetLineageGraphRequest.serializeBinaryToWriter = function(mess /** - * optional LineageGraphQueryOptions options = 1; - * @return {?proto.ml_metadata.LineageGraphQueryOptions} + * optional LineageSubgraphQueryOptions lineage_subgraph_query_options = 1; + * @return {?proto.ml_metadata.LineageSubgraphQueryOptions} */ -proto.ml_metadata.GetLineageGraphRequest.prototype.getOptions = function() { - return /** @type{?proto.ml_metadata.LineageGraphQueryOptions} */ ( - jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.LineageGraphQueryOptions, 1)); +proto.ml_metadata.GetLineageSubgraphRequest.prototype.getLineageSubgraphQueryOptions = function() { + return /** @type{?proto.ml_metadata.LineageSubgraphQueryOptions} */ ( + jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.LineageSubgraphQueryOptions, 1)); }; /** - * @param {?proto.ml_metadata.LineageGraphQueryOptions|undefined} value - * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this + * @param {?proto.ml_metadata.LineageSubgraphQueryOptions|undefined} value + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} returns this */ -proto.ml_metadata.GetLineageGraphRequest.prototype.setOptions = function(value) { +proto.ml_metadata.GetLineageSubgraphRequest.prototype.setLineageSubgraphQueryOptions = function(value) { return jspb.Message.setWrapperField(this, 1, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} returns this */ -proto.ml_metadata.GetLineageGraphRequest.prototype.clearOptions = function() { - return this.setOptions(undefined); +proto.ml_metadata.GetLineageSubgraphRequest.prototype.clearLineageSubgraphQueryOptions = function() { + return this.setLineageSubgraphQueryOptions(undefined); }; @@ -19992,16 +26082,53 @@ proto.ml_metadata.GetLineageGraphRequest.prototype.clearOptions = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetLineageGraphRequest.prototype.hasOptions = function() { +proto.ml_metadata.GetLineageSubgraphRequest.prototype.hasLineageSubgraphQueryOptions = function() { return jspb.Message.getField(this, 1) != null; }; +/** + * optional google.protobuf.FieldMask read_mask = 3; + * @return {?proto.google.protobuf.FieldMask} + */ +proto.ml_metadata.GetLineageSubgraphRequest.prototype.getReadMask = function() { + return /** @type{?proto.google.protobuf.FieldMask} */ ( + jspb.Message.getWrapperField(this, google_protobuf_field_mask_pb.FieldMask, 3)); +}; + + +/** + * @param {?proto.google.protobuf.FieldMask|undefined} value + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} returns this +*/ +proto.ml_metadata.GetLineageSubgraphRequest.prototype.setReadMask = function(value) { + return jspb.Message.setWrapperField(this, 3, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} returns this + */ +proto.ml_metadata.GetLineageSubgraphRequest.prototype.clearReadMask = function() { + return this.setReadMask(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.ml_metadata.GetLineageSubgraphRequest.prototype.hasReadMask = function() { + return jspb.Message.getField(this, 3) != null; +}; + + /** * optional TransactionOptions transaction_options = 2; * @return {?proto.ml_metadata.TransactionOptions} */ -proto.ml_metadata.GetLineageGraphRequest.prototype.getTransactionOptions = function() { +proto.ml_metadata.GetLineageSubgraphRequest.prototype.getTransactionOptions = function() { return /** @type{?proto.ml_metadata.TransactionOptions} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.TransactionOptions, 2)); }; @@ -20009,18 +26136,18 @@ proto.ml_metadata.GetLineageGraphRequest.prototype.getTransactionOptions = funct /** * @param {?proto.ml_metadata.TransactionOptions|undefined} value - * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} returns this */ -proto.ml_metadata.GetLineageGraphRequest.prototype.setTransactionOptions = function(value) { +proto.ml_metadata.GetLineageSubgraphRequest.prototype.setTransactionOptions = function(value) { return jspb.Message.setWrapperField(this, 2, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetLineageGraphRequest} returns this + * @return {!proto.ml_metadata.GetLineageSubgraphRequest} returns this */ -proto.ml_metadata.GetLineageGraphRequest.prototype.clearTransactionOptions = function() { +proto.ml_metadata.GetLineageSubgraphRequest.prototype.clearTransactionOptions = function() { return this.setTransactionOptions(undefined); }; @@ -20029,7 +26156,7 @@ proto.ml_metadata.GetLineageGraphRequest.prototype.clearTransactionOptions = fun * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetLineageGraphRequest.prototype.hasTransactionOptions = function() { +proto.ml_metadata.GetLineageSubgraphRequest.prototype.hasTransactionOptions = function() { return jspb.Message.getField(this, 2) != null; }; @@ -20050,8 +26177,8 @@ if (jspb.Message.GENERATE_TO_OBJECT) { * http://goto/soy-param-migration * @return {!Object} */ -proto.ml_metadata.GetLineageGraphResponse.prototype.toObject = function(opt_includeInstance) { - return proto.ml_metadata.GetLineageGraphResponse.toObject(opt_includeInstance, this); +proto.ml_metadata.GetLineageSubgraphResponse.prototype.toObject = function(opt_includeInstance) { + return proto.ml_metadata.GetLineageSubgraphResponse.toObject(opt_includeInstance, this); }; @@ -20060,13 +26187,13 @@ proto.ml_metadata.GetLineageGraphResponse.prototype.toObject = function(opt_incl * @param {boolean|undefined} includeInstance Deprecated. Whether to include * the JSPB instance for transitional soy proto support: * http://goto/soy-param-migration - * @param {!proto.ml_metadata.GetLineageGraphResponse} msg The msg instance to transform. + * @param {!proto.ml_metadata.GetLineageSubgraphResponse} msg The msg instance to transform. * @return {!Object} * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetLineageGraphResponse.toObject = function(includeInstance, msg) { +proto.ml_metadata.GetLineageSubgraphResponse.toObject = function(includeInstance, msg) { var f, obj = { - subgraph: (f = msg.getSubgraph()) && ml_metadata_proto_metadata_store_pb.LineageGraph.toObject(includeInstance, f) + lineageSubgraph: (f = msg.getLineageSubgraph()) && ml_metadata_proto_metadata_store_pb.LineageGraph.toObject(includeInstance, f) }; if (includeInstance) { @@ -20080,23 +26207,23 @@ proto.ml_metadata.GetLineageGraphResponse.toObject = function(includeInstance, m /** * Deserializes binary data (in protobuf wire format). * @param {jspb.ByteSource} bytes The bytes to deserialize. - * @return {!proto.ml_metadata.GetLineageGraphResponse} + * @return {!proto.ml_metadata.GetLineageSubgraphResponse} */ -proto.ml_metadata.GetLineageGraphResponse.deserializeBinary = function(bytes) { +proto.ml_metadata.GetLineageSubgraphResponse.deserializeBinary = function(bytes) { var reader = new jspb.BinaryReader(bytes); - var msg = new proto.ml_metadata.GetLineageGraphResponse; - return proto.ml_metadata.GetLineageGraphResponse.deserializeBinaryFromReader(msg, reader); + var msg = new proto.ml_metadata.GetLineageSubgraphResponse; + return proto.ml_metadata.GetLineageSubgraphResponse.deserializeBinaryFromReader(msg, reader); }; /** * Deserializes binary data (in protobuf wire format) from the * given reader into the given message object. - * @param {!proto.ml_metadata.GetLineageGraphResponse} msg The message object to deserialize into. + * @param {!proto.ml_metadata.GetLineageSubgraphResponse} msg The message object to deserialize into. * @param {!jspb.BinaryReader} reader The BinaryReader to use. - * @return {!proto.ml_metadata.GetLineageGraphResponse} + * @return {!proto.ml_metadata.GetLineageSubgraphResponse} */ -proto.ml_metadata.GetLineageGraphResponse.deserializeBinaryFromReader = function(msg, reader) { +proto.ml_metadata.GetLineageSubgraphResponse.deserializeBinaryFromReader = function(msg, reader) { while (reader.nextField()) { if (reader.isEndGroup()) { break; @@ -20106,7 +26233,7 @@ proto.ml_metadata.GetLineageGraphResponse.deserializeBinaryFromReader = function case 1: var value = new ml_metadata_proto_metadata_store_pb.LineageGraph; reader.readMessage(value,ml_metadata_proto_metadata_store_pb.LineageGraph.deserializeBinaryFromReader); - msg.setSubgraph(value); + msg.setLineageSubgraph(value); break; default: reader.skipField(); @@ -20121,9 +26248,9 @@ proto.ml_metadata.GetLineageGraphResponse.deserializeBinaryFromReader = function * Serializes the message to binary data (in protobuf wire format). * @return {!Uint8Array} */ -proto.ml_metadata.GetLineageGraphResponse.prototype.serializeBinary = function() { +proto.ml_metadata.GetLineageSubgraphResponse.prototype.serializeBinary = function() { var writer = new jspb.BinaryWriter(); - proto.ml_metadata.GetLineageGraphResponse.serializeBinaryToWriter(this, writer); + proto.ml_metadata.GetLineageSubgraphResponse.serializeBinaryToWriter(this, writer); return writer.getResultBuffer(); }; @@ -20131,13 +26258,13 @@ proto.ml_metadata.GetLineageGraphResponse.prototype.serializeBinary = function() /** * Serializes the given message to binary data (in protobuf wire * format), writing to the given BinaryWriter. - * @param {!proto.ml_metadata.GetLineageGraphResponse} message + * @param {!proto.ml_metadata.GetLineageSubgraphResponse} message * @param {!jspb.BinaryWriter} writer * @suppress {unusedLocalVariables} f is only used for nested messages */ -proto.ml_metadata.GetLineageGraphResponse.serializeBinaryToWriter = function(message, writer) { +proto.ml_metadata.GetLineageSubgraphResponse.serializeBinaryToWriter = function(message, writer) { var f = undefined; - f = message.getSubgraph(); + f = message.getLineageSubgraph(); if (f != null) { writer.writeMessage( 1, @@ -20149,10 +26276,10 @@ proto.ml_metadata.GetLineageGraphResponse.serializeBinaryToWriter = function(mes /** - * optional LineageGraph subgraph = 1; + * optional LineageGraph lineage_subgraph = 1; * @return {?proto.ml_metadata.LineageGraph} */ -proto.ml_metadata.GetLineageGraphResponse.prototype.getSubgraph = function() { +proto.ml_metadata.GetLineageSubgraphResponse.prototype.getLineageSubgraph = function() { return /** @type{?proto.ml_metadata.LineageGraph} */ ( jspb.Message.getWrapperField(this, ml_metadata_proto_metadata_store_pb.LineageGraph, 1)); }; @@ -20160,19 +26287,19 @@ proto.ml_metadata.GetLineageGraphResponse.prototype.getSubgraph = function() { /** * @param {?proto.ml_metadata.LineageGraph|undefined} value - * @return {!proto.ml_metadata.GetLineageGraphResponse} returns this + * @return {!proto.ml_metadata.GetLineageSubgraphResponse} returns this */ -proto.ml_metadata.GetLineageGraphResponse.prototype.setSubgraph = function(value) { +proto.ml_metadata.GetLineageSubgraphResponse.prototype.setLineageSubgraph = function(value) { return jspb.Message.setWrapperField(this, 1, value); }; /** * Clears the message field making it undefined. - * @return {!proto.ml_metadata.GetLineageGraphResponse} returns this + * @return {!proto.ml_metadata.GetLineageSubgraphResponse} returns this */ -proto.ml_metadata.GetLineageGraphResponse.prototype.clearSubgraph = function() { - return this.setSubgraph(undefined); +proto.ml_metadata.GetLineageSubgraphResponse.prototype.clearLineageSubgraph = function() { + return this.setLineageSubgraph(undefined); }; @@ -20180,7 +26307,7 @@ proto.ml_metadata.GetLineageGraphResponse.prototype.clearSubgraph = function() { * Returns whether this field is set. * @return {boolean} */ -proto.ml_metadata.GetLineageGraphResponse.prototype.hasSubgraph = function() { +proto.ml_metadata.GetLineageSubgraphResponse.prototype.hasLineageSubgraph = function() { return jspb.Message.getField(this, 1) != null; }; diff --git a/go.mod b/go.mod index be3525416f..452e75f520 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( github.com/aws/aws-sdk-go v1.44.239 github.com/cenkalti/backoff v2.2.1+incompatible github.com/eapache/go-resiliency v1.2.0 + github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a // indirect + github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect github.com/fsnotify/fsnotify v1.5.4 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/go-openapi/errors v0.20.2 @@ -24,18 +26,20 @@ require ( github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 + github.com/jackc/pgx/v5 v5.4.2 github.com/jinzhu/gorm v1.9.1 github.com/jinzhu/inflection v1.0.0 // indirect github.com/kubeflow/kfp-tekton/tekton-catalog/pipeline-loops v0.0.0-20230502223014-2dc9314d8d35 github.com/kubeflow/pipelines/api v0.0.0-20230331215358-758c91f76784 github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20230404213301-bd9f74e34de6 - github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20220118175555-e78ed557ddcb + github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 github.com/lestrrat-go/strftime v1.0.4 github.com/mattn/go-sqlite3 v1.14.16 github.com/minio/minio-go/v6 v6.0.57 github.com/peterhellberg/duration v0.0.0-20191119133758-ec6baeebcd10 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.4.0 github.com/robfig/cron v1.2.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.10.1 @@ -43,7 +47,7 @@ require ( github.com/tektoncd/pipeline v0.47.3 go.uber.org/zap v1.24.0 gocloud.dev v0.22.0 - golang.org/x/net v0.9.0 + golang.org/x/net v0.10.0 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 google.golang.org/grpc v1.54.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 @@ -83,9 +87,7 @@ require ( github.com/colinmarc/hdfs v1.1.4-0.20180805212432-9746310a4d31 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/doublerebel/bellows v0.0.0-20160303004610-f177d92a03d3 // indirect - github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/go-kit/log v0.2.0 // indirect @@ -115,6 +117,8 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jcmturner/gofork v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -144,7 +148,6 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/statsd_exporter v0.21.0 // indirect @@ -163,13 +166,13 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/automaxprocs v1.4.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.8.0 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect golang.org/x/mod v0.9.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index ded0764748..7796ef311e 100644 --- a/go.sum +++ b/go.sum @@ -1837,6 +1837,7 @@ github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bY github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -1847,6 +1848,8 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -1857,6 +1860,8 @@ github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9 github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ= +github.com/jackc/pgx/v5 v5.4.2 h1:u1gmGDwbdRUZiwisBm/Ky2M14uQyUP65bG8+20nnyrg= +github.com/jackc/pgx/v5 v5.4.2/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -1982,8 +1987,8 @@ github.com/kubeflow/pipelines/api v0.0.0-20230331215358-758c91f76784 h1:ZVCoqnKn github.com/kubeflow/pipelines/api v0.0.0-20230331215358-758c91f76784/go.mod h1:T7TOQB36gGe97yUdfVAnYK5uuT0+uQbLNHDUHxYkmE4= github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20230404213301-bd9f74e34de6 h1:ApWW5ZH45ruvQCmkp7RewHlPKGwqBNSSRxEHGJFiAOA= github.com/kubeflow/pipelines/kubernetes_platform v0.0.0-20230404213301-bd9f74e34de6/go.mod h1:CJkKr356RlpZP/gQRuHf3Myrn1qJtoUVe4EMCmtwarg= -github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20220118175555-e78ed557ddcb h1:i0RzcKBlfGHueIwrUlKB+AvVZPuMUJIYe1g8nvhwgbo= -github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20220118175555-e78ed557ddcb/go.mod h1:chIDffBaVQ/asNl1pTTdbAymYcuBKf8BR3YtSP+3FEU= +github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800 h1:YAW+X9xCW8Yq5tQaBBQaLTNU9CJj8Nr7lx1+k66ZHJ0= +github.com/kubeflow/pipelines/third_party/ml-metadata v0.0.0-20230810215105-e1f0c010f800/go.mod h1:chIDffBaVQ/asNl1pTTdbAymYcuBKf8BR3YtSP+3FEU= github.com/kulti/thelper v0.4.0/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U= github.com/kunwardeep/paralleltest v1.0.3/go.mod h1:vLydzomDFpk7yu5UX02RmP0H8QfRPOV/oFhWN85Mjb4= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= @@ -2338,8 +2343,9 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -2822,8 +2828,9 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2987,8 +2994,9 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -3211,8 +3219,9 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -3226,8 +3235,9 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/hack/update-all-requirements.sh b/hack/update-all-requirements.sh index 3a71bf81e9..0b4cd533ba 100755 --- a/hack/update-all-requirements.sh +++ b/hack/update-all-requirements.sh @@ -18,6 +18,7 @@ set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" REPO_ROOT="${DIR}/.." -cd "${REPO_ROOT}/backend" && bash update_requirements.sh cd "${REPO_ROOT}/backend/src/apiserver/visualization" && bash update_requirements.sh cd "${REPO_ROOT}/test/sample-test/hack" && bash update_requirements.sh +cd "${REPO_ROOT}/backend/metadata_writer" && bash update_requirements.sh +cd "${REPO_ROOT}/backend" && bash update_requirements.sh diff --git a/hack/update-requirements.sh b/hack/update-requirements.sh index 68361a3b3d..d7879ab26a 100755 --- a/hack/update-requirements.sh +++ b/hack/update-requirements.sh @@ -18,7 +18,7 @@ set -euo pipefail IMAGE=${1:-"python:3.7"} docker run -i --rm --entrypoint "" "$IMAGE" sh -c ' - python3 -m pip install pip setuptools --upgrade --quiet - python3 -m pip install pip-tools==5.4.0 --quiet - pip-compile --verbose --output-file - - + python3 -m pip install pip setuptools --quiet --upgrade + python3 -m pip install pip-tools==6.14.0 --quiet + pip-compile --resolver=backtracking --output-file - - ' diff --git a/kubernetes_platform/OWNERS b/kubernetes_platform/OWNERS index ba9cb373cb..59bb717307 100644 --- a/kubernetes_platform/OWNERS +++ b/kubernetes_platform/OWNERS @@ -2,9 +2,7 @@ approvers: - chensun - connor-mccarthy - gkcalat - - Linchin reviewers: - chensun - connor-mccarthy - gkcalat - - Linchin diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml index fd778769e4..a563a4844f 100644 --- a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml +++ b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/application.yaml @@ -12,7 +12,7 @@ metadata: spec: descriptor: type: Kubeflow Pipelines - version: 2.0.1 + version: 2.0.3 description: |- Reusable end-to-end ML workflow maintainers: diff --git a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml index 92e961b525..3807b02fb8 100644 --- a/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml +++ b/manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/pipeline.yaml @@ -149,6 +149,19 @@ rules: - get - list - watch + - apiGroups: + - pipelines.kubeflow.org + resources: + - scheduledworkflows + - workflows + verbs: + - report + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get - apiGroups: - '' resources: @@ -560,7 +573,18 @@ spec: requests: cpu: 120m memory: 500Mi + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens + name: persistenceagent-sa-token serviceAccountName: ml-pipeline-persistenceagent + volumes: + - name: persistenceagent-sa-token + projected: + sources: + - serviceAccountToken: + path: persistenceagent-sa-token + expirationSeconds: 3600 + audience: pipelines.kubeflow.org --- apiVersion: apps/v1 kind: Deployment diff --git a/manifests/gcp_marketplace/schema.yaml b/manifests/gcp_marketplace/schema.yaml index 46cd2b385e..fa50b0207d 100644 --- a/manifests/gcp_marketplace/schema.yaml +++ b/manifests/gcp_marketplace/schema.yaml @@ -1,9 +1,9 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: 2.0.1 + publishedVersion: 2.0.3 publishedVersionMetadata: - releaseNote: Based on 2.0.1 version. + releaseNote: Based on 2.0.3 version. releaseTypes: - Feature recommended: false @@ -121,6 +121,9 @@ x-google-marketplace: - apiGroups: ['kubeflow.org'] resources: ['*'] verbs: ['*'] + - apiGroups: ['pipelines.kubeflow.org'] + resources: ['*'] + verbs: ['*'] - apiGroups: [''] resources: ['configmaps', 'events', 'pods/log', 'persistentvolumes', 'persistentvolumeclaims', 'persistentvolumeclaims/finalizers', 'pods', 'pods/exec'] verbs: ['*'] diff --git a/manifests/gcp_marketplace/test/snapshot-base.yaml b/manifests/gcp_marketplace/test/snapshot-base.yaml index 7769633694..426ac6b01f 100644 --- a/manifests/gcp_marketplace/test/snapshot-base.yaml +++ b/manifests/gcp_marketplace/test/snapshot-base.yaml @@ -1353,6 +1353,19 @@ rules: - get - list - watch + - apiGroups: + - pipelines.kubeflow.org + resources: + - scheduledworkflows + - workflows + verbs: + - report + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get - apiGroups: - '' resources: @@ -2410,7 +2423,18 @@ spec: requests: cpu: 120m memory: 500Mi + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens + name: persistenceagent-sa-token serviceAccountName: ml-pipeline-persistenceagent + volumes: + - name: persistenceagent-sa-token + projected: + sources: + - serviceAccountToken: + path: persistenceagent-sa-token + expirationSeconds: 3600 + audience: pipelines.kubeflow.org --- # Source: kubeflow-pipelines/templates/pipeline.yaml apiVersion: apps/v1 @@ -2714,7 +2738,7 @@ metadata: spec: descriptor: type: Kubeflow Pipelines - version: 2.0.0-beta.1 + version: 2.0.1 description: |- Reusable end-to-end ML workflow maintainers: diff --git a/manifests/gcp_marketplace/test/snapshot-emissary.yaml b/manifests/gcp_marketplace/test/snapshot-emissary.yaml index 7769633694..426ac6b01f 100644 --- a/manifests/gcp_marketplace/test/snapshot-emissary.yaml +++ b/manifests/gcp_marketplace/test/snapshot-emissary.yaml @@ -1353,6 +1353,19 @@ rules: - get - list - watch + - apiGroups: + - pipelines.kubeflow.org + resources: + - scheduledworkflows + - workflows + verbs: + - report + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get - apiGroups: - '' resources: @@ -2410,7 +2423,18 @@ spec: requests: cpu: 120m memory: 500Mi + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens + name: persistenceagent-sa-token serviceAccountName: ml-pipeline-persistenceagent + volumes: + - name: persistenceagent-sa-token + projected: + sources: + - serviceAccountToken: + path: persistenceagent-sa-token + expirationSeconds: 3600 + audience: pipelines.kubeflow.org --- # Source: kubeflow-pipelines/templates/pipeline.yaml apiVersion: apps/v1 @@ -2714,7 +2738,7 @@ metadata: spec: descriptor: type: Kubeflow Pipelines - version: 2.0.0-beta.1 + version: 2.0.1 description: |- Reusable end-to-end ML workflow maintainers: diff --git a/manifests/gcp_marketplace/test/snapshot-managed-storage-with-db-prefix.yaml b/manifests/gcp_marketplace/test/snapshot-managed-storage-with-db-prefix.yaml index 8ade0429dd..488067d4a0 100644 --- a/manifests/gcp_marketplace/test/snapshot-managed-storage-with-db-prefix.yaml +++ b/manifests/gcp_marketplace/test/snapshot-managed-storage-with-db-prefix.yaml @@ -1352,6 +1352,19 @@ rules: - get - list - watch + - apiGroups: + - pipelines.kubeflow.org + resources: + - scheduledworkflows + - workflows + verbs: + - report + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get - apiGroups: - '' resources: @@ -2451,7 +2464,18 @@ spec: requests: cpu: 120m memory: 500Mi + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens + name: persistenceagent-sa-token serviceAccountName: ml-pipeline-persistenceagent + volumes: + - name: persistenceagent-sa-token + projected: + sources: + - serviceAccountToken: + path: persistenceagent-sa-token + expirationSeconds: 3600 + audience: pipelines.kubeflow.org --- # Source: kubeflow-pipelines/templates/pipeline.yaml apiVersion: apps/v1 @@ -2764,7 +2788,7 @@ metadata: spec: descriptor: type: Kubeflow Pipelines - version: 2.0.0-beta.1 + version: 2.0.1 description: |- Reusable end-to-end ML workflow maintainers: diff --git a/manifests/gcp_marketplace/test/snapshot-managed-storage.yaml b/manifests/gcp_marketplace/test/snapshot-managed-storage.yaml index a36ef81590..f6abd1a557 100644 --- a/manifests/gcp_marketplace/test/snapshot-managed-storage.yaml +++ b/manifests/gcp_marketplace/test/snapshot-managed-storage.yaml @@ -1352,6 +1352,19 @@ rules: - get - list - watch + - apiGroups: + - pipelines.kubeflow.org + resources: + - scheduledworkflows + - workflows + verbs: + - report + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get - apiGroups: - '' resources: @@ -2451,7 +2464,18 @@ spec: requests: cpu: 120m memory: 500Mi + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens + name: persistenceagent-sa-token serviceAccountName: ml-pipeline-persistenceagent + volumes: + - name: persistenceagent-sa-token + projected: + sources: + - serviceAccountToken: + path: persistenceagent-sa-token + expirationSeconds: 3600 + audience: pipelines.kubeflow.org --- # Source: kubeflow-pipelines/templates/pipeline.yaml apiVersion: apps/v1 @@ -2764,7 +2788,7 @@ metadata: spec: descriptor: type: Kubeflow Pipelines - version: 2.0.0-beta.1 + version: 2.0.1 description: |- Reusable end-to-end ML workflow maintainers: diff --git a/manifests/kustomize/base/cache-deployer/kustomization.yaml b/manifests/kustomize/base/cache-deployer/kustomization.yaml index a9640aa6cb..de44a30c52 100644 --- a/manifests/kustomize/base/cache-deployer/kustomization.yaml +++ b/manifests/kustomize/base/cache-deployer/kustomization.yaml @@ -8,4 +8,4 @@ commonLabels: app: cache-deployer images: - name: gcr.io/ml-pipeline/cache-deployer - newTag: 2.0.1 + newTag: 2.0.3 diff --git a/manifests/kustomize/base/cache/kustomization.yaml b/manifests/kustomize/base/cache/kustomization.yaml index 56c40afbe4..24fa04023a 100644 --- a/manifests/kustomize/base/cache/kustomization.yaml +++ b/manifests/kustomize/base/cache/kustomization.yaml @@ -2,12 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cache-deployment.yaml - - cache-service.yaml - cache-role.yaml - cache-rolebinding.yaml - cache-sa.yaml + - cache-service.yaml commonLabels: app: cache-server images: - name: gcr.io/ml-pipeline/cache-server - newTag: 2.0.1 + newTag: 2.0.3 diff --git a/manifests/kustomize/base/installs/generic/kustomization.yaml b/manifests/kustomize/base/installs/generic/kustomization.yaml index e07e3ee33c..5eb52f95a8 100644 --- a/manifests/kustomize/base/installs/generic/kustomization.yaml +++ b/manifests/kustomize/base/installs/generic/kustomization.yaml @@ -45,4 +45,4 @@ vars: fieldref: fieldpath: data.defaultPipelineRoot configurations: -- params.yaml + - params.yaml diff --git a/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml b/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml index 47ce6b687d..cd3b48e8ee 100644 --- a/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml +++ b/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml @@ -11,9 +11,12 @@ data: until the changes take effect. A quick way to restart all deployments in a namespace: `kubectl rollout restart deployment -n `. appName: pipeline - appVersion: 2.0.1 - dbHost: mysql - dbPort: "3306" + appVersion: 2.0.3 + dbHost: mysql # relic to be removed after release + dbPort: "3306" # relic to be removed after release + dbType: mysql + mysqlHost: mysql + mysqlPort: "3306" mlmdDb: metadb cacheDb: cachedb pipelineDb: mlpipeline diff --git a/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml b/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml new file mode 100644 index 0000000000..21ca80694a --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/kustomization.yaml @@ -0,0 +1,48 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow +bases: +- ../../../postgresql/pipeline +- ../../../postgresql/cache +- ../../../cache-deployer +resources: +- pipeline-install-config.yaml +- postgres-secret-extended.yaml +vars: +- name: kfp-namespace + objref: + kind: Deployment + apiVersion: apps/v1 + name: ml-pipeline + fieldref: + fieldpath: metadata.namespace +- name: kfp-app-name + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.appName +- name: kfp-app-version + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.appVersion +- name: kfp-artifact-bucket-name + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.bucketName +- name: kfp-default-pipeline-root + objref: + kind: ConfigMap + name: pipeline-install-config + apiVersion: v1 + fieldref: + fieldpath: data.defaultPipelineRoot +configurations: + - params.yaml diff --git a/manifests/kustomize/base/installs/generic/postgres/params.yaml b/manifests/kustomize/base/installs/generic/postgres/params.yaml new file mode 100644 index 0000000000..9b8a7efd52 --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/params.yaml @@ -0,0 +1,10 @@ +# Allow Kustomize var to replace following fields. +varReference: +- path: data/config + kind: ConfigMap +- path: data/defaultPipelineRoot + kind: ConfigMap +- path: metadata/name + kind: Application +- path: spec/descriptor/version + kind: Application diff --git a/manifests/kustomize/base/installs/generic/postgres/pipeline-install-config.yaml b/manifests/kustomize/base/installs/generic/postgres/pipeline-install-config.yaml new file mode 100644 index 0000000000..f5d87c6797 --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/pipeline-install-config.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pipeline-install-config +data: + warning: | + 1. Do not use kubectl to edit this configmap, because some values are used + during kustomize build. Instead, change the configmap and apply the entire + kustomize manifests again. + 2. After updating the configmap, some deployments may need to be restarted + until the changes take effect. A quick way to restart all deployments in a + namespace: `kubectl rollout restart deployment -n `. + appName: pipeline + appVersion: 2.0.0 + dbHost: postgres # relic to be removed after release + dbPort: "5432" # relic to be removed after release + dbType: postgres + postgresHost: postgres + postgresPort: "5432" + mlmdDb: metadb + cacheDb: cachedb + pipelineDb: mlpipeline + bucketName: mlpipeline + ## defaultPipelineRoot: Optional. Default pipeline root in v2 compatible mode. + ## https://www.kubeflow.org/docs/components/pipelines/sdk/v2/v2-compatibility/ + ## + ## If the field is not set, kfp-launcher configmaps won't be created and + ## v2 compatible mode defaults to minio://mlpipeline/v2/artifacts as pipeline + ## root. + ## + ## When not in Kubeflow Pipelines multi-user mode, the config works as you + ## would normally expect. + ## + ## In Kubeflow Pipelines multi-user mode, the config creates default + ## kfp-launcher configmaps in each user's namespace. Users can edit the + ## kfp-launcher configmap's defaultPipelineRoot field afterwards to configure + ## namespace-specific default pipeline root. The namespace specific changes in + ## kfp-launcher configmap won't be overridden by pipeline-install-config. + ## + ## Caveat: when you update the config from a non-empty value, only new + ## namespaces get the updated config by default. Owners of existing namespaces + ## must delete the kfp-launcher configmap to get the new default config value. + ## + ## Examples: + ## defaultPipelineRoot: minio://mlpipeline/v2/artifacts + ## defaultPipelineRoot: gs://your-bucket/path/to/artifacts + ## defaultPipelineRoot: s3://your-bucket/path/to/artifacts + ## + ## V2 Compatible Mode Feature stage: + ## [Beta](https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#beta) + defaultPipelineRoot: "" + ## autoUpdatePipelineDefaultVersion: States if the pipeline version + ## should be updated by defult for a versioned pipeline or not when a new + ## version is uploaded. This sets the deployment wide definition. + autoUpdatePipelineDefaultVersion: "true" + ## cronScheduleTimezone: States the timezone which should be used for + ## the cron scheduler. If not specified the local timezone of the + ## cluster will be used. Valid values are UTC, Local or values according to + ## the IANA Time Zone database, such as "America/New_York" and "Asia/Shanghai". + ## Feature stage: + ## [Alpha](https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#alpha) + cronScheduleTimezone: "UTC" + ## cacheImage is the image that the mutating webhook will use to patch + ## cached steps with. Will be used to echo a message announcing that + ## the cached step result will be used. If not set it will default to + ## 'gcr.io/google-containers/busybox' + cacheImage: "gcr.io/google-containers/busybox" + ## cacheNodeRestrictions the dummy container runing if output is cached + ## will run with the same affinity and node selector as the default pipeline + ## step. This is defaulted to 'false' to allow the pod to be scheduled on + ## any node and avoid defaulting to specific nodes. Allowed values are: + ## 'false' and 'true'. + cacheNodeRestrictions: "false" + ## MAXIMUM_CACHE_STALENESS configures caching according to + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching/ and + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching-v2/. + ## Larger than MAXIMUM_CACHE_STALENESS per pipeline user set values are + ## reduced to MAXIMUM_CACHE_STALENESS. + ## The administrator of the storage backend can rely on it to delete old cache + ## artifacts. + MAXIMUM_CACHE_STALENESS: "" + ## MAXIMUM_CACHE_STALENESS: "P30D" + ## DEFAULT_CACHE_STALENESS configures caching according to + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching/ and + ## https://www.kubeflow.org/docs/components/pipelines/overview/caching-v2/. + ## This value is used if the user did not set a value in the pipeline. + DEFAULT_CACHE_STALENESS: "" + ## DEFAULT_CACHE_STALENESS: "P7D" + ## ConMaxLifeTime will set the connection max lifetime for MySQL + ## this is very important to setup when using external databases. + ## See this issue for more details: https://github.com/kubeflow/pipelines/issues/5329 + ## Note: this value should be a string that can be parsed by `time.ParseDuration`. + ## If this value doesn't include a unit abbreviation, the units will be assumed + ## to be nanoseconds. + ConMaxLifeTime: "120s" diff --git a/manifests/kustomize/base/installs/generic/postgres/postgres-secret-extended.yaml b/manifests/kustomize/base/installs/generic/postgres/postgres-secret-extended.yaml new file mode 100644 index 0000000000..b67369ba1c --- /dev/null +++ b/manifests/kustomize/base/installs/generic/postgres/postgres-secret-extended.yaml @@ -0,0 +1,7 @@ +kind: Secret +apiVersion: v1 +metadata: + name: postgres-secret-extended +stringData: + username: user + password: "password" diff --git a/manifests/kustomize/base/installs/multi-user/persistence-agent/cluster-role.yaml b/manifests/kustomize/base/installs/multi-user/persistence-agent/cluster-role.yaml index 2df34121ba..0d2321241a 100644 --- a/manifests/kustomize/base/installs/multi-user/persistence-agent/cluster-role.yaml +++ b/manifests/kustomize/base/installs/multi-user/persistence-agent/cluster-role.yaml @@ -48,4 +48,11 @@ rules: - watch - update - patch - - delete \ No newline at end of file + - delete +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - reportMetrics + - readArtifact diff --git a/manifests/kustomize/base/installs/multi-user/persistence-agent/deployment-patch.yaml b/manifests/kustomize/base/installs/multi-user/persistence-agent/deployment-patch.yaml index a5e7a9fc26..1e165def42 100644 --- a/manifests/kustomize/base/installs/multi-user/persistence-agent/deployment-patch.yaml +++ b/manifests/kustomize/base/installs/multi-user/persistence-agent/deployment-patch.yaml @@ -7,14 +7,7 @@ spec: spec: containers: - name: ml-pipeline-persistenceagent - envFrom: - - configMapRef: - name: persistenceagent-config env: - name: NAMESPACE value: '' valueFrom: null - - name: KUBEFLOW_USERID_HEADER - value: kubeflow-userid - - name: KUBEFLOW_USERID_PREFIX - value: "" \ No newline at end of file diff --git a/manifests/kustomize/base/installs/multi-user/persistence-agent/kustomization.yaml b/manifests/kustomize/base/installs/multi-user/persistence-agent/kustomization.yaml index 560e0fc893..b1f65469e1 100644 --- a/manifests/kustomize/base/installs/multi-user/persistence-agent/kustomization.yaml +++ b/manifests/kustomize/base/installs/multi-user/persistence-agent/kustomization.yaml @@ -3,7 +3,3 @@ kind: Kustomization resources: - cluster-role.yaml - cluster-role-binding.yaml -configMapGenerator: -- name: persistenceagent-config - envs: - - params.env \ No newline at end of file diff --git a/manifests/kustomize/base/installs/multi-user/persistence-agent/params.env b/manifests/kustomize/base/installs/multi-user/persistence-agent/params.env deleted file mode 100644 index 4c3bab70f9..0000000000 --- a/manifests/kustomize/base/installs/multi-user/persistence-agent/params.env +++ /dev/null @@ -1 +0,0 @@ -MULTIUSER=true diff --git a/manifests/kustomize/base/metadata/base/kustomization.yaml b/manifests/kustomize/base/metadata/base/kustomization.yaml index 400315b107..218758b6d6 100644 --- a/manifests/kustomize/base/metadata/base/kustomization.yaml +++ b/manifests/kustomize/base/metadata/base/kustomization.yaml @@ -10,4 +10,4 @@ resources: namespace: kubeflow images: - name: gcr.io/ml-pipeline/metadata-envoy - newTag: 2.0.1 + newTag: 2.0.3 diff --git a/manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml b/manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml index 4ed344491d..1ea10488d6 100644 --- a/manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml +++ b/manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml @@ -22,7 +22,7 @@ spec: # * .cloudbuild.yaml and .release.cloudbuild.yaml # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml # * test/tag_for_hosted.sh - image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0 + image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 env: - name: DBCONFIG_USER valueFrom: diff --git a/manifests/kustomize/base/metadata/overlays/postgres/kustomization.yaml b/manifests/kustomize/base/metadata/overlays/postgres/kustomization.yaml new file mode 100644 index 0000000000..9f78bf3bbc --- /dev/null +++ b/manifests/kustomize/base/metadata/overlays/postgres/kustomization.yaml @@ -0,0 +1,38 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kubeflow + +bases: +- ../../base +resources: +- metadata-db-pvc.yaml +- metadata-db-deployment.yaml +- metadata-db-service.yaml + +patchesStrategicMerge: +- patches/metadata-grpc-deployment.yaml + +configMapGenerator: +- name: metadata-postgres-db-parameters + envs: + - params.env +secretGenerator: +- name: metadata-postgres-db-secrets + envs: + - secrets.env +generatorOptions: + disableNameSuffixHash: true + +images: +- name: postgres + newName: postgres + newTag: 14.7-alpine3.17 + +vars: +- name: MLMD_DB_HOST + objref: + kind: Service + name: metadata-postgres-db + apiVersion: v1 + fieldref: + fieldpath: metadata.name diff --git a/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-deployment.yaml b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-deployment.yaml new file mode 100644 index 0000000000..061d109e1e --- /dev/null +++ b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metadata-postgres-db + labels: + component: db +spec: + selector: + matchLabels: + component: db + replicas: 1 + strategy: + type: Recreate + template: + metadata: + name: db + labels: + component: db + annotations: + sidecar.istio.io/inject: "false" + spec: + containers: + - name: db-container + image: postgres + env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + ports: + - name: postgres + containerPort: 5432 + volumeMounts: + - name: metadata-postgres + mountPath: /var/lib/postgresql/data + volumes: + - name: metadata-postgres + persistentVolumeClaim: + claimName: metadata-postgres + diff --git a/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-pvc.yaml b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-pvc.yaml new file mode 100644 index 0000000000..13790489fa --- /dev/null +++ b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: metadata-postgres +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-service.yaml b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-service.yaml new file mode 100644 index 0000000000..63902a6661 --- /dev/null +++ b/manifests/kustomize/base/metadata/overlays/postgres/metadata-db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: metadata-postgres-db + labels: + component: db +spec: + type: ClusterIP + ports: + - port: 5432 + protocol: TCP + name: postgres + selector: + component: db diff --git a/manifests/kustomize/base/metadata/overlays/postgres/params.env b/manifests/kustomize/base/metadata/overlays/postgres/params.env new file mode 100644 index 0000000000..fce7e26772 --- /dev/null +++ b/manifests/kustomize/base/metadata/overlays/postgres/params.env @@ -0,0 +1,2 @@ +POSTGRES_PORT=5432 +POSTGRES_DBNAME=mlmdpostgres \ No newline at end of file diff --git a/manifests/kustomize/base/metadata/overlays/postgres/patches/metadata-grpc-deployment.yaml b/manifests/kustomize/base/metadata/overlays/postgres/patches/metadata-grpc-deployment.yaml new file mode 100644 index 0000000000..9f3c052b7b --- /dev/null +++ b/manifests/kustomize/base/metadata/overlays/postgres/patches/metadata-grpc-deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metadata-grpc-deployment +spec: + template: + spec: + containers: + - name: container + # Remove existing environment variables + env: + - $patch: replace + envFrom: + - configMapRef: + name: metadata-postgres-db-parameters + - secretRef: + name: metadata-postgres-db-secrets + - configMapRef: + name: metadata-grpc-configmap + args: ["--grpc_port=$(METADATA_GRPC_SERVICE_PORT)", + "--metadata_source_config_type=postgresql", + "--postgres_config_host=$(MLMD_DB_HOST)", + "--postgres_config_port=$(POSTGRES_PORT)", + "--postgres_config_dbname=$(POSTGRES_DBNAME)", + "--postgres_config_user=$(POSTGRES_USER)", + "--postgres_config_password=$(POSTGRES_PASSWORD)", + # "--postgres_config_skip_db_creation=true", + "--enable_database_upgrade=true"] diff --git a/manifests/kustomize/base/metadata/overlays/postgres/secrets.env b/manifests/kustomize/base/metadata/overlays/postgres/secrets.env new file mode 100644 index 0000000000..973d158283 --- /dev/null +++ b/manifests/kustomize/base/metadata/overlays/postgres/secrets.env @@ -0,0 +1,2 @@ +POSTGRES_USER=root +POSTGRES_PASSWORD=password \ No newline at end of file diff --git a/manifests/kustomize/base/pipeline/kustomization.yaml b/manifests/kustomize/base/pipeline/kustomization.yaml index d04099e521..e3d3ce55a2 100644 --- a/manifests/kustomize/base/pipeline/kustomization.yaml +++ b/manifests/kustomize/base/pipeline/kustomization.yaml @@ -37,14 +37,14 @@ resources: - kfp-launcher-configmap.yaml images: - name: gcr.io/ml-pipeline/api-server - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/persistenceagent - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/scheduledworkflow - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/frontend - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/viewer-crd-controller - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/visualization-server - newTag: 2.0.1 + newTag: 2.0.3 diff --git a/manifests/kustomize/base/pipeline/metadata-writer/kustomization.yaml b/manifests/kustomize/base/pipeline/metadata-writer/kustomization.yaml index 2d118e33cf..b503511088 100644 --- a/manifests/kustomize/base/pipeline/metadata-writer/kustomization.yaml +++ b/manifests/kustomize/base/pipeline/metadata-writer/kustomization.yaml @@ -7,4 +7,4 @@ resources: - metadata-writer-sa.yaml images: - name: gcr.io/ml-pipeline/metadata-writer - newTag: 2.0.1 + newTag: 2.0.3 diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index 7a37c7f478..754adf9057 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -33,6 +33,7 @@ spec: configMapKeyRef: name: pipeline-install-config key: bucketName + # relic variables - name: DBCONFIG_USER valueFrom: secretKeyRef: @@ -58,11 +59,44 @@ spec: configMapKeyRef: name: pipeline-install-config key: dbPort + # end of relic variables - name: DBCONFIG_CONMAXLIFETIME valueFrom: configMapKeyRef: name: pipeline-install-config key: ConMaxLifeTime + - name: DB_DRIVER_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbType + # MySQL Config + - name: DBCONFIG_MYSQLCONFIG_USER + valueFrom: + secretKeyRef: + name: mysql-secret + key: username + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-secret + key: password + - name: DBCONFIG_MYSQLCONFIG_DBNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: pipelineDb + - name: DBCONFIG_MYSQLCONFIG_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: mysqlHost + - name: DBCONFIG_MYSQLCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: mysqlPort + # end of MySQL Config - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-role.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-role.yaml index e95df98b88..ca22474e2c 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-role.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-role.yaml @@ -48,3 +48,10 @@ rules: - update - patch - delete +- apiGroups: + - pipelines.kubeflow.org + resources: + - runs + verbs: + - reportMetrics + - readArtifact \ No newline at end of file diff --git a/manifests/kustomize/base/postgresql/cache/cache-deployment-patch.yaml b/manifests/kustomize/base/postgresql/cache/cache-deployment-patch.yaml new file mode 100644 index 0000000000..324925767a --- /dev/null +++ b/manifests/kustomize/base/postgresql/cache/cache-deployment-patch.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cache-server +spec: + template: + spec: + containers: + - name: server + env: + - $patch: replace + - name: DEFAULT_CACHE_STALENESS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: DEFAULT_CACHE_STALENESS + - name: MAXIMUM_CACHE_STALENESS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: MAXIMUM_CACHE_STALENESS + - name: CACHE_IMAGE + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: cacheImage + - name: CACHE_NODE_RESTRICTIONS + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: cacheNodeRestrictions + - name: DBCONFIG_DRIVER + value: pgx + - name: DBCONFIG_DB_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: cacheDb + - name: DBCONFIG_HOST_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresHost + - name: DBCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresPort + - name: DBCONFIG_USER + valueFrom: + secretKeyRef: + name: postgres-secret-extended + key: username + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret-extended + key: password + - name: NAMESPACE_TO_WATCH + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # If you update WEBHOOK_PORT, also change the value of the + # containerPort "webhook-api" to match. + - name: WEBHOOK_PORT + value: "8443" + args: ["--db_driver=$(DBCONFIG_DRIVER)", + "--db_host=$(DBCONFIG_HOST_NAME)", + "--db_port=$(DBCONFIG_PORT)", + "--db_name=$(DBCONFIG_DB_NAME)", + "--db_user=$(DBCONFIG_USER)", + "--db_password=$(DBCONFIG_PASSWORD)", + "--namespace_to_watch=$(NAMESPACE_TO_WATCH)", + "--listen_port=$(WEBHOOK_PORT)", + ] diff --git a/manifests/kustomize/base/postgresql/cache/kustomization.yaml b/manifests/kustomize/base/postgresql/cache/kustomization.yaml new file mode 100644 index 0000000000..d4935432cc --- /dev/null +++ b/manifests/kustomize/base/postgresql/cache/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../cache +patchesStrategicMerge: +- cache-deployment-patch.yaml diff --git a/manifests/kustomize/base/postgresql/pipeline/kustomization.yaml b/manifests/kustomize/base/postgresql/pipeline/kustomization.yaml new file mode 100644 index 0000000000..3f87400eb3 --- /dev/null +++ b/manifests/kustomize/base/postgresql/pipeline/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: + - ../../pipeline +patchesStrategicMerge: + - ml-pipeline-apiserver-deployment-patch.yaml diff --git a/manifests/kustomize/base/postgresql/pipeline/ml-pipeline-apiserver-deployment-patch.yaml b/manifests/kustomize/base/postgresql/pipeline/ml-pipeline-apiserver-deployment-patch.yaml new file mode 100644 index 0000000000..7d621691c3 --- /dev/null +++ b/manifests/kustomize/base/postgresql/pipeline/ml-pipeline-apiserver-deployment-patch.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-pipeline +spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - $patch: replace + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: autoUpdatePipelineDefaultVersion + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: OBJECTSTORECONFIG_BUCKETNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: bucketName + - name: DBCONFIG_CONMAXLIFETIME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: ConMaxLifeTime + - name: DB_DRIVER_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: dbType + # PostgreSQL Config + - name: DBCONFIG_POSTGRESQLCONFIG_USER + valueFrom: + secretKeyRef: + name: postgres-secret-extended + key: username + - name: DBCONFIG_POSTGRESQLCONFIG_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret-extended + key: password + - name: DBCONFIG_POSTGRESQLCONFIG_DBNAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: pipelineDb + - name: DBCONFIG_POSTGRESQLCONFIG_HOST + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresHost + - name: DBCONFIG_POSTGRESQLCONFIG_PORT + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: postgresPort + # end of PostgreSQL variables + - name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey \ No newline at end of file diff --git a/manifests/kustomize/env/dev/postgresql/kustomization.yaml b/manifests/kustomize/env/dev/postgresql/kustomization.yaml new file mode 100644 index 0000000000..25aec96ef0 --- /dev/null +++ b/manifests/kustomize/env/dev/postgresql/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + # Application controller is used to provide Google Cloud Console integration. + - ../../../third-party/application + - ../../../base/application + - ../../platform-agnostic-postgresql + - ../../gcp/inverse-proxy + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please refer sample/cluster-scoped-resources to update the namespace for cluster-scoped-resources +namespace: kubeflow diff --git a/manifests/kustomize/env/gcp/inverse-proxy/kustomization.yaml b/manifests/kustomize/env/gcp/inverse-proxy/kustomization.yaml index e8798ef205..827f961978 100644 --- a/manifests/kustomize/env/gcp/inverse-proxy/kustomization.yaml +++ b/manifests/kustomize/env/gcp/inverse-proxy/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: gcr.io/ml-pipeline/inverse-proxy-agent - newTag: 2.0.1 + newTag: 2.0.3 resources: - proxy-configmap.yaml - proxy-deployment.yaml diff --git a/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml new file mode 100644 index 0000000000..00a9d4613b --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-postgresql/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: + - ../../base/installs/generic/postgres + - ../../base/metadata/overlays/postgres + - ../../third-party/argo/installs/namespace + - ../../third-party/minio/base + - ../../third-party/postgresql/base + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow diff --git a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml index 346a61348e..ae0681064c 100644 --- a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml +++ b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml @@ -15,13 +15,13 @@ commonLabels: images: - name: gcr.io/ml-pipeline/api-server newName: quay.io/aipipeline/apiserver - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/persistenceagent newName: quay.io/aipipeline/persistenceagent - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/scheduledworkflow newName: quay.io/aipipeline/scheduledworkflow - newTag: 2.0.1 + newTag: 2.0.3 patches: - patch: |- diff --git a/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml index 261d647aed..4414deb8c7 100644 --- a/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml +++ b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml @@ -13,13 +13,13 @@ resources: images: - name: gcr.io/ml-pipeline/api-server newName: quay.io/aipipeline/apiserver - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/persistenceagent newName: quay.io/aipipeline/persistenceagent - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/scheduledworkflow newName: quay.io/aipipeline/scheduledworkflow - newTag: 2.0.1 + newTag: 2.0.3 labels: - includeSelectors: true diff --git a/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml index dd2024cb65..b48a5efb69 100644 --- a/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml +++ b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml @@ -64,13 +64,13 @@ patches: images: - name: gcr.io/ml-pipeline/api-server newName: quay.io/aipipeline/apiserver - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/persistenceagent newName: quay.io/aipipeline/persistenceagent - newTag: 2.0.1 + newTag: 2.0.3 - name: gcr.io/ml-pipeline/scheduledworkflow newName: quay.io/aipipeline/scheduledworkflow - newTag: 2.0.1 + newTag: 2.0.3 labels: - includeSelectors: true diff --git a/manifests/kustomize/third-party/postgresql/README.md b/manifests/kustomize/third-party/postgresql/README.md new file mode 100644 index 0000000000..c0c1cda433 --- /dev/null +++ b/manifests/kustomize/third-party/postgresql/README.md @@ -0,0 +1,15 @@ +## Build PostgreSQL yaml + +```bash +# In this folder of manifests/kustomize/third-party/postgresql +rm -rf build +mkdir build +kustomize build ./base -o build +``` + +## Deploy PostgreSQL container + +```bash +# In this folder of manifests/kustomize/third-party/postgresql +kubectl -n apply -f build +``` \ No newline at end of file diff --git a/manifests/kustomize/third-party/postgresql/base/kustomization.yaml b/manifests/kustomize/third-party/postgresql/base/kustomization.yaml new file mode 100644 index 0000000000..e7951ef9f4 --- /dev/null +++ b/manifests/kustomize/third-party/postgresql/base/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- pg-deployment.yaml +- pg-pvc.yaml +- pg-service.yaml +- pg-secret.yaml +- pg-serviceaccount.yaml diff --git a/manifests/kustomize/third-party/postgresql/base/pg-deployment.yaml b/manifests/kustomize/third-party/postgresql/base/pg-deployment.yaml new file mode 100644 index 0000000000..bd0bf4baaf --- /dev/null +++ b/manifests/kustomize/third-party/postgresql/base/pg-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + labels: + app: postgres +spec: + selector: + matchLabels: + app: postgres + strategy: + type: Recreate + template: + metadata: + labels: + app: postgres + spec: + serviceAccountName: postgresql + containers: + - image: postgres:14.7-alpine3.17 + name: postgres + env: + - name: POSTGRES_DB + value: postgres + - name: POSTGRES_USER + value: user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: root_password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + name: postgres + readinessProbe: + exec: + command: ["psql", "-U", "user", "-d", "postgres", "-c", "SELECT 1"] + initialDelaySeconds: 15 + timeoutSeconds: 2 + volumeMounts: + - name: postgres-stateful-data + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-stateful-data + persistentVolumeClaim: + claimName: postgres-pvc diff --git a/manifests/kustomize/third-party/postgresql/base/pg-pvc.yaml b/manifests/kustomize/third-party/postgresql/base/pg-pvc.yaml new file mode 100644 index 0000000000..b59616a73c --- /dev/null +++ b/manifests/kustomize/third-party/postgresql/base/pg-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc + labels: + app: postgres +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi \ No newline at end of file diff --git a/manifests/kustomize/third-party/postgresql/base/pg-secret.yaml b/manifests/kustomize/third-party/postgresql/base/pg-secret.yaml new file mode 100644 index 0000000000..22648465f3 --- /dev/null +++ b/manifests/kustomize/third-party/postgresql/base/pg-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret +type: Opaque +data: + root_password: password \ No newline at end of file diff --git a/manifests/kustomize/third-party/postgresql/base/pg-service.yaml b/manifests/kustomize/third-party/postgresql/base/pg-service.yaml new file mode 100644 index 0000000000..002eff982d --- /dev/null +++ b/manifests/kustomize/third-party/postgresql/base/pg-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-service + labels: + app: postgres +spec: + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 + selector: + app: postgres diff --git a/manifests/kustomize/third-party/postgresql/base/pg-serviceaccount.yaml b/manifests/kustomize/third-party/postgresql/base/pg-serviceaccount.yaml new file mode 100644 index 0000000000..4397c93c10 --- /dev/null +++ b/manifests/kustomize/third-party/postgresql/base/pg-serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: postgresql diff --git a/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml b/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml index 4f2887b07a..55a89408c5 100644 --- a/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml +++ b/manifests/kustomize/third-party/tekton-custom-task/kustomization.yaml @@ -16,16 +16,16 @@ images: newTag: 1.7.1 - name: kfp-v2-dev-driver-controller newName: quay.io/aipipeline/tekton-driver - newTag: 2.0.1 + newTag: 2.0.3 - name: tekton-exithandler-controller newName: quay.io/aipipeline/tekton-exithandler-controller - newTag: 2.0.1 + newTag: 2.0.3 - name: tekton-exithandler-webhook newName: quay.io/aipipeline/tekton-exithandler-webhook - newTag: 2.0.1 + newTag: 2.0.3 - name: tekton-kfptask-controller newName: quay.io/aipipeline/tekton-kfptask-controller - newTag: 2.0.1 + newTag: 2.0.3 - name: tekton-kfptask-webhook newName: quay.io/aipipeline/tekton-kfptask-webhook - newTag: 2.0.1 + newTag: 2.0.3 diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..a079fdd1c7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +addopts = --ignore=sdk/python/kfp/deprecated --ignore=sdk/python/kfp/tests +testpaths = sdk/python/kfp diff --git a/samples/contrib/intel-oneapi-samples/README.md b/samples/contrib/intel-oneapi-samples/README.md new file mode 100644 index 0000000000..451a1f2787 --- /dev/null +++ b/samples/contrib/intel-oneapi-samples/README.md @@ -0,0 +1,164 @@ +

+ Intel Logo +

+ +# Intel® Optimized XGBoost daal4py Kubeflow Pipeline + +This example demonstrates how to optimize an XGBoost Kubeflow Pipeline using a sample +dataset to predict the probability of loan default. +The reference solution enables the use of the +[Intel® Optimization for XGBoost*](https://www.intel.com/content/www/us/en/developer/tools/oneapi/optimization-for-xgboost.html), +[Intel® oneAPI Data Analytics Library (Intel® oneDAL)](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onedal.html), +and [Intel® Extension for Scikit-Learn*](https://www.intel.com/content/www/us/en/developer/tools/oneapi/scikit-learn.html) +to accelerate an end-to-end training and inference XGBoost pipeline. + +## Table of Contents +- [System Requirements](#system-requirements) +- [Overview](#pipeline-overview) +- [Pipeline Optimizations](#pipeline-optimizations) +- [Pipeline Parameters](#pipeline-parameters) +- [Pipeline Results](#pipeline-results) + +## System Requirements + +- Before running the code for the pipeline, please ensure you have downloaded and installed +[Kubeflow Pipelines SDK](https://v1-5-branch.kubeflow.org/docs/components/pipelines/sdk-v2/) +v2.0.1 or above. +- To attain the most performance benefits from the Intel software optimizations, deploy the +pipeline on a 3rd or 4th Generation [Intel® Xeon® Processor](https://www.intel.com/content/www/us/en/products/details/processors/xeon.html). + +## Pipeline Overview + +This pipeline is derived from the +[Loan Default Risk Prediction AI Reference Kit](https://github.com/oneapi-src/loan-default-risk-prediction). +The code has been enhanced through refactoring to achieve better modularity and suitability for +Kubeflow Pipelines. The credit risk data set used in the pipeline is obtained from +[Kaggle](https://www.kaggle.com/datasets/laotse/credit-risk-dataset)* +and synthetically augmented for testing and benchmarking purposes. Below is a graph of the full +XGBoost daal4py Kubeflow Pipeline. + +

+ Intel XGBoost daal4py Pipeline +

+ +The pipeline consists of the following seven components: +- **Load data**: This component loads the dataset (`credit_risk_dataset.csv`) from the URL specified +in the pipeline run parameters and performs synthetic data augmentation. +- **Create training and test sets**: This component splits the data into training and test sets of an +approximately 75:25 split for model evaluation. +- **Preprocess features**: This component transforms the categorical features of the training and +test sets by using one-hot encoding, imputes missing values, and power-transforms numerical features. +- **Train XGBoost model**: This component trains an XGBoost model using the accelerations provided by +the Intel Optimizations for XGBoost. +- **Convert XGBoost model to daal4py**: This component converts the XGBoost model to an +inference-optimized daal4py classifier. +- **daal4py Inference**: This component computes predictions using the inference-optimized daal4py +classifier and evaluates model performance. It returns a summary of the precision, recall, and F1 +score for each class, as well as the area under the curve (AUC) and accuracy score of the model. +- **Plot ROC Curve**: This component performs model validation on the test data and generates a +graph of the receiver operating characteristic (ROC) curve. + +[Back to Table of Contents](#table-of-contents) + +## Pipeline Optimizations + +#### Enable the Intel Optimization for XGBoost + +The [XGBoost optimizations](https://www.intel.com/content/www/us/en/developer/tools/oneapi/optimization-for-xgboost.html) +for training and inference on CPUs are upstreamed into the open source XGBoost framework. +Ensure you are using the latest version of XGBoost to access the most Intel optimizations. +The following code sample is implemented in the `train_xgboost_model` component. + +``` +dtrain = xgb.DMatrix(X_train.values, y_train.values) + +# define model parameters +params = { + "objective": "binary:logistic", + "eval_metric": "logloss", + "nthread": 4, # num_cpu + "tree_method": "hist", + "learning_rate": 0.02, + "max_depth": 10, + "min_child_weight": 6, + "n_jobs": 4, # num_cpu, + "verbosity": 1} + +# train XGBoost model +clf = xgb.train(params = params, + dtrain = dtrain, + num_boost_round = 500) +``` + +#### Convert the Trained XGBoost Model to daal4py + +[daal4py](https://www.intel.com/content/www/us/en/developer/articles/guide/a-daal4py-introduction-and-getting-started-guide.html) +is the Python API of the oneAPI Data Analytics Library, oneDAL. daal4py helps to further +optimize model prediction, or inference, on CPUs. The following code demonstrates how to +convert a trained XGBoost model into daal4py format and calculate the predicted +classification results, implemented in the `convert_xgboost_to_daal4py` and `daal4py_inference` +components. + +``` +# convert XGBoost model to daal4py +daal_model = d4p.get_gbt_model_from_xgboost(clf) + + +# compute class labels and probabilities +daal_prediction = d4p.gbt_classification_prediction( + nClasses = 2, + resultsToEvaluate = "computeClassLabels|computeClassProbabilities" +).compute(X_test, daal_model) +``` + +#### Enable the Intel Extension for Scikit-Learn +The [Intel Extension for Scikit-Learn](https://www.intel.com/content/www/us/en/developer/tools/oneapi/scikit-learn.html) +provides CPU accelerations for many scikit-learn libraries. Below is an example +using the scikit-learn extension to accelerate the computation of the ROC curve. +The following code is implemented in the `plot_roc_curve` component. + +``` +# call patch_sklearn() before importing scikit-learn libraries +from sklearnex import patch_sklearn +patch_sklearn() +from sklearn.metrics import roc_curve + + +# calculate the ROC curve using the CPU-accelerated version +fpr, tpr, thresholds = roc_curve( + y_true = prediction_data['y_test'], + y_score = prediction_data['y_prob'], + pos_label = 1) +``` + +[Back to Table of Contents](#table-of-contents) + +## Pipeline Parameters + +The XGBoost daal4py Kubeflow Pipeline consists of the following two parameters: +- `data_url`: The sample dataset can be downloaded from +[Kaggle](https://www.kaggle.com/datasets/laotse/credit-risk-dataset) +and hosted on a public URL of your choice. +- `data_size`: The recommended data size for the pipeline is 1 million. + +## Pipeline Results + +When the Pipeline tasks `daal4py-inference` and `plot-roc-curve` are finished running, +click on the Visualization tab of the `metrics` and `roc_curve_daal4py` artifacts to +view the model performance results. You should see a similar graph of the receiver +operating characteristic (ROC) curve as the one below. + +

+ ROC Curve +

+ +[Back to Table of Contents](#table-of-contents) + +## Next Steps + +Thanks for checking out this tutorial! If you would like to implement this reference solution +on a cloud service provider like AWS, Azure, or GCP, you can view the full deployment steps, +as well as additional Intel® Optimized Cloud Modules +[here](https://www.intel.com/content/www/us/en/developer/topic-technology/cloud-optimization.html). + +[Back to Table of Contents](#table-of-contents) \ No newline at end of file diff --git a/samples/contrib/intel-oneapi-samples/assets/intel-xgb-d4p-pipeline-roc-curve.png b/samples/contrib/intel-oneapi-samples/assets/intel-xgb-d4p-pipeline-roc-curve.png new file mode 100644 index 0000000000..73d0b530ac Binary files /dev/null and b/samples/contrib/intel-oneapi-samples/assets/intel-xgb-d4p-pipeline-roc-curve.png differ diff --git a/samples/contrib/intel-oneapi-samples/assets/intel-xgb-d4p-pipeline.png b/samples/contrib/intel-oneapi-samples/assets/intel-xgb-d4p-pipeline.png new file mode 100644 index 0000000000..d826cb2a76 Binary files /dev/null and b/samples/contrib/intel-oneapi-samples/assets/intel-xgb-d4p-pipeline.png differ diff --git a/samples/contrib/intel-oneapi-samples/assets/logo-classicblue-800px.png b/samples/contrib/intel-oneapi-samples/assets/logo-classicblue-800px.png new file mode 100644 index 0000000000..c9f1c81722 Binary files /dev/null and b/samples/contrib/intel-oneapi-samples/assets/logo-classicblue-800px.png differ diff --git a/samples/contrib/intel-oneapi-samples/intel_xgboost_daal4py_pipeline.py b/samples/contrib/intel-oneapi-samples/intel_xgboost_daal4py_pipeline.py new file mode 100644 index 0000000000..377722136c --- /dev/null +++ b/samples/contrib/intel-oneapi-samples/intel_xgboost_daal4py_pipeline.py @@ -0,0 +1,502 @@ +from kfp import dsl +from kfp import compiler +from kfp.dsl import (Input, Output, Dataset, Model, Metrics, ClassificationMetrics) + +@dsl.component( + base_image="python:3.10", + packages_to_install=["numpy", "pandas", "loguru"]) +def load_data( + data_url: str, + data_size: int, + credit_risk_dataset: Output[Dataset]): + + ''' + Downloads credit_risk_dataset.csv file and generates + additional synthetic data for benchmarking and testing purposes. + + Input Parameters + ---------------- + data_url : str + url where the dataset is hosted + data_size : int + size of final dataset desired, default 1M rows + + Output Artifacts + ---------------- + credit_risk_dataset : Dataset + data that has been synthetically augmented or loaded from URL provided + ''' + + import numpy as np + import pandas as pd + from loguru import logger + + logger.info("Loading csv from {}", data_url) + data = pd.read_csv(data_url) + logger.info("Done!") + + # number of rows to generate + if data_size < data.shape[0]: + pass + else: + logger.info("Generating {:,} rows of data...", data_size) + repeats = data_size // len(data) + data = data.loc[np.repeat(data.index.values, repeats + 1)] + data = data.iloc[:data_size] + + # perturbing all int/float columns + person_age = data["person_age"].values + np.random.randint( + -1, 1, size=len(data) + ) + person_income = data["person_income"].values + np.random.normal( + 0, 10, size=len(data) + ) + person_emp_length = data[ + "person_emp_length" + ].values + np.random.randint(-1, 1, size=len(data)) + loan_amnt = data["loan_amnt"].values + np.random.normal( + 0, 5, size=len(data) + ) + loan_int_rate = data["loan_int_rate"].values + np.random.normal( + 0, 0.2, size=len(data) + ) + loan_percent_income = data["loan_percent_income"].values + ( + np.random.randint(0, 100, size=len(data)) / 1000 + ) + cb_person_cred_hist_length = data[ + "cb_person_cred_hist_length" + ].values + np.random.randint(0, 2, size=len(data)) + + # perturbing all binary columns + perturb_idx = np.random.rand(len(data)) > 0.1 + random_values = np.random.choice( + data["person_home_ownership"].unique(), len(data) + ) + person_home_ownership = np.where( + perturb_idx, data["person_home_ownership"], random_values + ) + perturb_idx = np.random.rand(len(data)) > 0.1 + random_values = np.random.choice( + data["loan_intent"].unique(), len(data) + ) + loan_intent = np.where(perturb_idx, data["loan_intent"], random_values) + perturb_idx = np.random.rand(len(data)) > 0.1 + random_values = np.random.choice( + data["loan_grade"].unique(), len(data) + ) + loan_grade = np.where(perturb_idx, data["loan_grade"], random_values) + perturb_idx = np.random.rand(len(data)) > 0.1 + random_values = np.random.choice( + data["cb_person_default_on_file"].unique(), len(data) + ) + cb_person_default_on_file = np.where( + perturb_idx, data["cb_person_default_on_file"], random_values + ) + data = pd.DataFrame( + list( + zip( + person_age, + person_income, + person_home_ownership, + person_emp_length, + loan_intent, + loan_grade, + loan_amnt, + loan_int_rate, + data["loan_status"].values, + loan_percent_income, + cb_person_default_on_file, + cb_person_cred_hist_length, + ) + ), + columns = data.columns, + ) + + data = data.drop_duplicates() + assert len(data) == data_size + data.reset_index(drop = True) + + data.to_csv(credit_risk_dataset.path, index = None) + +@dsl.component( + base_image="python:3.10", + packages_to_install=["pandas", "scikit-learn", "loguru"]) +def create_train_test_set( + data: Input[Dataset], + x_train_data: Output[Dataset], + y_train_data: Output[Dataset], + x_test_data: Output[Dataset], + y_test_data: Output[Dataset]): + + ''' + Creates 75:25 split of input dataset for model evaluation. + + Input Artifacts + --------------- + data : Dataset + dataset that has been synthetically augmented by the load_data() function + + Output Artifacts + ---------------- + x_train_data : Dataset + training features, 75% of original dataset + y_train_data : Dataset + training labels of target variable, loan_status + x_test_data : Dataset + test features, 25% of original dataset + y_test_data : Dataset + test labels of target variable, loan_status + ''' + + import pandas as pd + from loguru import logger + from sklearn.model_selection import train_test_split + + data = pd.read_csv(data.path) + + logger.info("Creating training and test sets...") + train, test = train_test_split(data, test_size = 0.25, random_state = 0) + + X_train = train.drop(["loan_status"], axis = 1) + y_train = train["loan_status"] + + X_test = test.drop(["loan_status"], axis = 1) + y_test = test["loan_status"] + + logger.info("Training and test sets created.\n" \ + "X_train size: {}, y_train size: {}\n" \ + "X_test size: {}, y_test size: {}", + X_train.shape, y_train.shape, X_test.shape, y_test.shape) + + X_train.to_csv(x_train_data.path, index = False) + y_train.to_csv(y_train_data.path, index = False, header = None) + X_test.to_csv(x_test_data.path, index = False) + y_test.to_csv(y_test_data.path, index = False, header = None) + +@dsl.component( + base_image="python:3.10", + packages_to_install=["pandas", "scikit-learn"]) +def preprocess_features( + x_train: Input[Dataset], + x_test: Input[Dataset], + x_train_processed: Output[Dataset], + x_test_processed: Output[Dataset]): + + ''' + Performs data preprocessing of training and test features. + + Input Artifacts + --------------- + x_train : Dataset + original unprocessed training features + x_test : Dataset + original unprocessed test features + + Output Artifacts + ---------------- + x_train_processed : Dataset + processed and scaled training features + x_test_processed : Dataset + processed and scaled test features + ''' + + import pandas as pd + from sklearn.compose import ColumnTransformer + from sklearn.impute import SimpleImputer + from sklearn.pipeline import Pipeline + from sklearn.preprocessing import OneHotEncoder, PowerTransformer + + X_train = pd.read_csv(x_train.path) + X_test = pd.read_csv(x_test.path) + + # data processing pipeline + num_imputer = Pipeline(steps=[("imputer", SimpleImputer(strategy = "median"))]) + pow_transformer = PowerTransformer() + cat_transformer = OneHotEncoder(handle_unknown = "ignore") + preprocessor = ColumnTransformer( + transformers = [ + ( + "num", + num_imputer, + [ + "loan_int_rate", + "person_emp_length", + "cb_person_cred_hist_length", + ], + ), + ( + "pow", + pow_transformer, + ["person_age", "person_income", "loan_amnt", "loan_percent_income"], + ), + ( + "cat", + cat_transformer, + [ + "person_home_ownership", + "loan_intent", + "loan_grade", + "cb_person_default_on_file", + ], + ), + ], + remainder="passthrough", + ) + + preprocess = Pipeline(steps = [("preprocessor", preprocessor)]) + + X_train = pd.DataFrame(preprocess.fit_transform(X_train)) + X_test = pd.DataFrame(preprocess.transform(X_test)) + + X_train.to_csv(x_train_processed.path, index = False, header = None) + X_test.to_csv(x_test_processed.path, index = False, header = None) + +@dsl.component( + base_image="python:3.10", + packages_to_install=["pandas", "xgboost", "joblib", "loguru"]) +def train_xgboost_model( + x_train: Input[Dataset], + y_train: Input[Dataset], + xgb_model: Output[Model]): + + ''' + Trains an XGBoost classification model. + + Input Artifacts + --------------- + x_train : Dataset + processed and scaled training features + y_train : Dataset + training labels of target variable, loan_status + + Output Artifacts + ---------------- + xgb_model : Model + trained XGBoost model + ''' + + import joblib + import pandas as pd + import xgboost as xgb + from loguru import logger + + X_train = pd.read_csv(x_train.path, header = None) + y_train = pd.read_csv(y_train.path, header = None) + + dtrain = xgb.DMatrix(X_train.values, y_train.values) + + # define model parameters + params = { + "objective": "binary:logistic", + "eval_metric": "logloss", + "nthread": 4, # num_cpu + "tree_method": "hist", + "learning_rate": 0.02, + "max_depth": 10, + "min_child_weight": 6, + "n_jobs": 4, # num_cpu, + "verbosity": 1 + } + + # train XGBoost model + logger.info("Training XGBoost model...") + clf = xgb.train(params = params, + dtrain = dtrain, + num_boost_round = 500) + + with open(xgb_model.path, "wb") as file_writer: + joblib.dump(clf, file_writer) + +@dsl.component( + base_image="python:3.10", + packages_to_install=["daal4py", "joblib", "loguru"]) +def convert_xgboost_to_daal4py( + xgb_model: Input[Model], + daal4py_model: Output[Model]): + + ''' + Converts XGBoost model to inference-optimized daal4py classifier. + + Input Artifacts + --------------- + xgb_model : Model + trained XGBoost classifier + + Output Artifacts + ---------------- + daal4py_model : Model + inference-optimized daal4py classifier + ''' + + import daal4py as d4p + import joblib + from loguru import logger + + with open(xgb_model.path, "rb") as file_reader: + clf = joblib.load(file_reader) + + logger.info("Converting XGBoost model to Daal4py...") + daal_model = d4p.get_gbt_model_from_xgboost(clf) + logger.info("Done!") + + with open(daal4py_model.path, "wb") as file_writer: + joblib.dump(daal_model, file_writer) + +@dsl.component( + base_image="python:3.10", + packages_to_install=["daal4py", "pandas", "scikit-learn", + "scikit-learn-intelex", "joblib"]) +def daal4py_inference( + x_test: Input[Dataset], + y_test: Input[Dataset], + daal4py_model: Input[Model], + prediction_data: Output[Dataset], + report: Output[Dataset], + metrics: Output[Metrics] +): + + ''' + Computes predictions using the inference-optimized daal4py classifier + and evaluates model performance. + + Input Artifacts + --------------- + x_test : Dataset + processed and scaled test features + y_test : Dataset + test labels of target variable, loan_status + daal4py_model : Model + inference-optimized daal4py classifier + + Output Artifacts + ---------------- + prediction_data : Dataset + dataset containing true test labels and predicted probabilities + report : Dataset + summary of the precision, recall, F1 score for each class + metrics : Metrics + scalar classification metrics containing the model's AUC and accuracy + ''' + + import daal4py as d4p + import joblib + import pandas as pd + + from sklearnex import patch_sklearn + patch_sklearn() + from sklearn.metrics import roc_auc_score, accuracy_score, classification_report + + X_test = pd.read_csv(x_test.path, header = None) + y_test = pd.read_csv(y_test.path, header = None) + + with open(daal4py_model.path, "rb") as file_reader: + daal_model = joblib.load(file_reader) + + daal_prediction = d4p.gbt_classification_prediction( + nClasses = 2, + resultsToEvaluate = "computeClassLabels|computeClassProbabilities" + ).compute(X_test, daal_model) + + y_pred = daal_prediction.prediction + y_prob = daal_prediction.probabilities[:,1] + + results = classification_report( + y_test, y_pred, + target_names = ["Non-Default", "Default"], + output_dict = True + ) + results = pd.DataFrame(results).transpose() + results.to_csv(report.path) + + auc = roc_auc_score(y_test, y_prob) + metrics.log_metric('AUC', auc) + + accuracy = (accuracy_score(y_test, y_pred)*100) + metrics.log_metric('Accuracy', accuracy) + + predictions = pd.DataFrame({'y_test': y_test.values.flatten(), + 'y_prob': y_prob}) + predictions.to_csv(prediction_data.path, index = False) + +@dsl.component( + base_image="python:3.10", + packages_to_install=["numpy", "pandas", "scikit-learn", + "scikit-learn-intelex"]) +def plot_roc_curve( + predictions: Input[Dataset], + class_metrics: Output[ClassificationMetrics] +): + + ''' + Function to plot Receiver Operating Characteristic (ROC) curve. + + Input Artifacts + --------------- + predictions : Dataset + dataset containing true test labels and predicted probabilities + + Output Artifacts + ---------------- + class_metrics : ClassificationMetrics + classification metrics containing fpr, tpr, and thresholds + ''' + + import pandas as pd + from numpy import inf + + from sklearnex import patch_sklearn + patch_sklearn() + from sklearn.metrics import roc_curve + + prediction_data = pd.read_csv(predictions.path) + + fpr, tpr, thresholds = roc_curve( + y_true = prediction_data['y_test'], + y_score = prediction_data['y_prob'], + pos_label = 1) + thresholds[thresholds == inf] = 0 + + class_metrics.log_roc_curve(fpr, tpr, thresholds) + +@dsl.pipeline +def intel_xgboost_daal4py_pipeline( + data_url: str, + data_size: int): + + load_data_op = load_data( + data_url = data_url, data_size = data_size + ) + + create_train_test_set_op = create_train_test_set( + data = load_data_op.outputs['credit_risk_dataset'] + ) + + preprocess_features_op = preprocess_features( + x_train = create_train_test_set_op.outputs['x_train_data'], + x_test = create_train_test_set_op.outputs['x_test_data'] + ) + + train_xgboost_model_op = train_xgboost_model( + x_train = preprocess_features_op.outputs['x_train_processed'], + y_train = create_train_test_set_op.outputs['y_train_data'] + ) + + convert_xgboost_to_daal4py_op = convert_xgboost_to_daal4py( + xgb_model = train_xgboost_model_op.outputs['xgb_model'] + ) + + daal4py_inference_op = daal4py_inference( + x_test = preprocess_features_op.outputs['x_test_processed'], + y_test = create_train_test_set_op.outputs['y_test_data'], + daal4py_model = convert_xgboost_to_daal4py_op.outputs['daal4py_model'] + ) + + plot_roc_curve_op = plot_roc_curve( + predictions = daal4py_inference_op.outputs['prediction_data'] + ) + +if __name__ == '__main__': + # Compiling the pipeline + compiler.Compiler().compile( + pipeline_func = intel_xgboost_daal4py_pipeline, + package_path = 'intel-xgboost-daal4py-pipeline.yaml') \ No newline at end of file diff --git a/contrib/samples/openvino/deployer/README.md b/samples/contrib/openvino/deployer/README.md similarity index 100% rename from contrib/samples/openvino/deployer/README.md rename to samples/contrib/openvino/deployer/README.md diff --git a/contrib/samples/openvino/deployer/component.yaml b/samples/contrib/openvino/deployer/component.yaml similarity index 100% rename from contrib/samples/openvino/deployer/component.yaml rename to samples/contrib/openvino/deployer/component.yaml diff --git a/contrib/samples/openvino/deployer/deployer.png b/samples/contrib/openvino/deployer/deployer.png similarity index 100% rename from contrib/samples/openvino/deployer/deployer.png rename to samples/contrib/openvino/deployer/deployer.png diff --git a/contrib/samples/openvino/deployer/deployer.py b/samples/contrib/openvino/deployer/deployer.py similarity index 100% rename from contrib/samples/openvino/deployer/deployer.py rename to samples/contrib/openvino/deployer/deployer.py diff --git a/contrib/samples/openvino/model_optimizer/README.md b/samples/contrib/openvino/model_optimizer/README.md similarity index 100% rename from contrib/samples/openvino/model_optimizer/README.md rename to samples/contrib/openvino/model_optimizer/README.md diff --git a/contrib/samples/openvino/model_optimizer/convert_model_pipeline.py b/samples/contrib/openvino/model_optimizer/convert_model_pipeline.py similarity index 100% rename from contrib/samples/openvino/model_optimizer/convert_model_pipeline.py rename to samples/contrib/openvino/model_optimizer/convert_model_pipeline.py diff --git a/contrib/samples/openvino/predict/README.md b/samples/contrib/openvino/predict/README.md similarity index 100% rename from contrib/samples/openvino/predict/README.md rename to samples/contrib/openvino/predict/README.md diff --git a/contrib/samples/openvino/predict/numpy_predict.py b/samples/contrib/openvino/predict/numpy_predict.py similarity index 100% rename from contrib/samples/openvino/predict/numpy_predict.py rename to samples/contrib/openvino/predict/numpy_predict.py diff --git a/contrib/samples/openvino/tf-slim/README.md b/samples/contrib/openvino/tf-slim/README.md similarity index 100% rename from contrib/samples/openvino/tf-slim/README.md rename to samples/contrib/openvino/tf-slim/README.md diff --git a/contrib/samples/openvino/tf-slim/demo_pipeline1.png b/samples/contrib/openvino/tf-slim/demo_pipeline1.png similarity index 100% rename from contrib/samples/openvino/tf-slim/demo_pipeline1.png rename to samples/contrib/openvino/tf-slim/demo_pipeline1.png diff --git a/contrib/samples/openvino/tf-slim/demo_pipeline2.png b/samples/contrib/openvino/tf-slim/demo_pipeline2.png similarity index 100% rename from contrib/samples/openvino/tf-slim/demo_pipeline2.png rename to samples/contrib/openvino/tf-slim/demo_pipeline2.png diff --git a/contrib/samples/openvino/tf-slim/demo_pipeline3.png b/samples/contrib/openvino/tf-slim/demo_pipeline3.png similarity index 100% rename from contrib/samples/openvino/tf-slim/demo_pipeline3.png rename to samples/contrib/openvino/tf-slim/demo_pipeline3.png diff --git a/contrib/samples/openvino/tf-slim/tf-slim.py b/samples/contrib/openvino/tf-slim/tf-slim.py similarity index 100% rename from contrib/samples/openvino/tf-slim/tf-slim.py rename to samples/contrib/openvino/tf-slim/tf-slim.py diff --git a/samples/core/XGBoost/xgboost_sample.py b/samples/core/XGBoost/xgboost_sample.py index d44cfaa73a..e11388a8a0 100644 --- a/samples/core/XGBoost/xgboost_sample.py +++ b/samples/core/XGBoost/xgboost_sample.py @@ -1,5 +1,5 @@ -import kfp.deprecated as kfp -from kfp.deprecated import components +import kfp as kfp +from kfp import components chicago_taxi_dataset_op = components.load_component_from_url( 'https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml' @@ -47,7 +47,7 @@ def xgboost_pipeline(): # Training and prediction on dataset in Apache Parquet format training_data_parquet = convert_csv_to_apache_parquet_op( - training_data_csv).output + data=training_data_csv).output model_trained_on_parquet = xgboost_train_on_parquet_op( training_data=training_data_parquet, diff --git a/samples/core/XGBoost/xgboost_sample_test.py b/samples/core/XGBoost/xgboost_sample_test.py index 44e2c2a6b7..8f43c17edc 100644 --- a/samples/core/XGBoost/xgboost_sample_test.py +++ b/samples/core/XGBoost/xgboost_sample_test.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp +import kfp as kfp from .xgboost_sample import xgboost_pipeline from kfp.samples.test.utils import run_pipeline_func, TestCase run_pipeline_func([ TestCase( pipeline_func=xgboost_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, + mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/core/caching/caching.ipynb b/samples/core/caching/caching.ipynb index 0a03f8e761..8efc1e6fbc 100644 --- a/samples/core/caching/caching.ipynb +++ b/samples/core/caching/caching.ipynb @@ -11,11 +11,11 @@ "import datetime\n", "import time\n", "\n", - "import kfp.deprecated as kfp\n", - "from kfp.deprecated.components import create_component_from_func\n", + "import kfp as kfp\n", + "from kfp import dsl\n", "\n", "\n", - "@create_component_from_func\n", + "@dsl.component\n", "def do_work_op(seconds: float = 60) -> str:\n", " import datetime\n", " import time\n", @@ -27,9 +27,11 @@ " return datetime.datetime.now().isoformat()\n", "\n", "\n", + "@kfp.dsl.pipeline(name='caching-pipeline')\n", "def caching_pipeline(seconds: float = 60):\n", " # All outputs of successful executions are cached\n", - " work_task = do_work_op(seconds)\n" + " work_task = do_work_op(seconds=seconds)\n", + "\n" ] }, { @@ -81,21 +83,14 @@ "metadata": {}, "outputs": [], "source": [ - "# Test 3\n", - "# For each task we can specify the maximum cached data staleness.\n", - "# For example: task.execution_options.caching_strategy.max_cache_staleness = \"P7D\" # (7 days)\n", - "# The `max_cache_staleness` attribute uses the [RFC3339 duration format](https://tools.ietf.org/html/rfc3339#appendix-A). For example: \"P0D\" (0 days), \"PT5H\" (5 hours; notice the \"T\")\n", - "# Cached results that are older than the specified time span, are not reused.\n", - "# In this case, the pipeline should not reuse the cached result, since they will be stale.\n", "\n", + "# Test 3\n", + "# In this case, the pipeline should not reuse the cached result, since they are disabled.\n", + "@kfp.dsl.pipeline(name='caching-pipeline3')\n", "def caching_pipeline3(seconds: float = 60):\n", - " # All outputs of successful executions are cached\n", - " work_task = do_work_op(seconds)\n", - " # TODO(Ark-kun): Fix handling non-zero periods in the backend\n", - " work_task.execution_options.caching_strategy.max_cache_staleness = 'P0D' # = Period: Time: 0 seconds\n", + " work_task = do_work_op(seconds=seconds)\n", + " work_task.set_caching_options(enable_caching=False)\n", "\n", - "# Waiting for some time for the cached data to become stale:\n", - "time.sleep(10)\n", "print(\"Starting test 3\")\n", "start_time = datetime.datetime.now()\n", "kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func(\n", diff --git a/samples/core/caching/caching_sample.py b/samples/core/caching/caching_sample.py index bd1220bddd..12ca3780d1 100644 --- a/samples/core/caching/caching_sample.py +++ b/samples/core/caching/caching_sample.py @@ -3,11 +3,11 @@ import datetime import time -import kfp.deprecated as kfp -from kfp.deprecated.components import create_component_from_func +import kfp as kfp +from kfp import dsl -@create_component_from_func +@dsl.component def do_work_op(seconds: float = 60) -> str: import datetime import time @@ -19,9 +19,10 @@ def do_work_op(seconds: float = 60) -> str: return datetime.datetime.now().isoformat() +@kfp.dsl.pipeline(name='caching-pipeline') def caching_pipeline(seconds: float = 60): # All outputs of successful executions are cached - work_task = do_work_op(seconds) + work_task = do_work_op(seconds=seconds) # Test 1 @@ -37,7 +38,6 @@ def caching_pipeline(seconds: float = 60): elapsed_time = datetime.datetime.now() - start_time print(f"Total run time: {int(elapsed_time.total_seconds())} seconds") - # Test 2 # Running the pipeline the second time. # The pipeline should reuse the cached results and complete faster. @@ -56,20 +56,14 @@ def caching_pipeline(seconds: float = 60): # Test 3 -# For each task we can specify the maximum cached data staleness. -# For example: task.execution_options.caching_strategy.max_cache_staleness = "P7D" # (7 days) -# The `max_cache_staleness` attribute uses the [RFC3339 duration format](https://tools.ietf.org/html/rfc3339#appendix-A). For example: "P0D" (0 days), "PT5H" (5 hours; notice the "T") -# Cached results that are older than the specified time span, are not reused. -# In this case, the pipeline should not reuse the cached result, since they will be stale. - +# In this case, the pipeline should not reuse the cached result, since they are +# disabled. +@kfp.dsl.pipeline(name='caching-pipeline3') def caching_pipeline3(seconds: float = 60): - # All outputs of successful executions are cached - work_task = do_work_op(seconds) - # TODO(Ark-kun): Fix handling non-zero periods in the backend - work_task.execution_options.caching_strategy.max_cache_staleness = 'P0D' # = Period: Time: 0 seconds + work_task = do_work_op(seconds=seconds) + work_task.set_caching_options(enable_caching=False) + -# Waiting for some time for the cached data to become stale: -time.sleep(10) print("Starting test 3") start_time = datetime.datetime.now() kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func( diff --git a/samples/core/caching/caching_test.py b/samples/core/caching/caching_test.py index 43c9970914..a81618e69b 100644 --- a/samples/core/caching/caching_test.py +++ b/samples/core/caching/caching_test.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp +import kfp as kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ TestCase( pipeline_file=relative_path(__file__, 'caching.ipynb'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - run_pipeline=False, + mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, + run_pipeline=False, ), ]) diff --git a/samples/core/execution_order/execution_order.py b/samples/core/execution_order/execution_order.py index 229364ccba..260ab14ab5 100755 --- a/samples/core/execution_order/execution_order.py +++ b/samples/core/execution_order/execution_order.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors +# Copyright 2019-2023 The Kubeflow Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,29 +14,29 @@ # limitations under the License. -from kfp.deprecated import dsl, compiler -import kfp.deprecated.components as comp +from kfp import compiler +from kfp import dsl -@comp.create_component_from_func +@dsl.component def echo1_op(text1: str): print(text1) -@comp.create_component_from_func +@dsl.component def echo2_op(text2: str): print(text2) @dsl.pipeline( - name='execution-order-pipeline', - description='A pipeline to demonstrate execution order management.' + name='execution-order-pipeline', + description='A pipeline to demonstrate execution order management.' ) def execution_order_pipeline(text1: str='message 1', text2: str='message 2'): """A two step pipeline with an explicitly defined execution order.""" - step1_task = echo1_op(text1) - step2_task = echo2_op(text2) + step1_task = echo1_op(text1=text1) + step2_task = echo2_op(text2=text2) step2_task.after(step1_task) if __name__ == '__main__': - compiler.Compiler().compile(execution_order_pipeline, __file__ + '.yaml') \ No newline at end of file + compiler.Compiler().compile(execution_order_pipeline, __file__ + '.yaml') diff --git a/samples/core/execution_order/execution_order_test.py b/samples/core/execution_order/execution_order_test.py index 64da9040c9..cc05c533e0 100644 --- a/samples/core/execution_order/execution_order_test.py +++ b/samples/core/execution_order/execution_order_test.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp +import kfp from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func run_pipeline_func([ TestCase( pipeline_file=relative_path(__file__, 'execution_order.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, + mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/core/exit_handler/exit_handler.py b/samples/core/exit_handler/exit_handler.py index eaf0effb7c..39bab79d72 100755 --- a/samples/core/exit_handler/exit_handler.py +++ b/samples/core/exit_handler/exit_handler.py @@ -12,41 +12,39 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Pipeline using ExitHandler.""" -from kfp import dsl, components, compiler -from kfp.components import InputPath, load_component_from_url +import os -gcs_download_op = load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/961b17fa6844e1d79e5d3686bb557d830d7b5a95/components/google-cloud/storage/download_blob/component.yaml' -) +from kfp import compiler +from kfp import dsl +from kfp.dsl import component -@components.create_component_from_func -def print_file(file_path: InputPath('Any')): - """Print a file.""" - with open(file_path) as f: - print(f.read()) +@component +def print_op(message: str): + """Prints a message.""" + print(message) -@components.create_component_from_func -def echo_msg(msg: str): - """Echo a message by parameter.""" - print(msg) +@component +def fail_op(message: str): + """Fails.""" + import sys + print(message) + sys.exit(1) -@dsl.pipeline( - name='exit-handler', - description= - 'Downloads a message and prints it. The exit handler will run after the pipeline finishes (successfully or not).' -) -def pipeline_exit_handler(url: str = 'gs://ml-pipeline/shakespeare1.txt'): - """A sample pipeline showing exit handler.""" - exit_task = echo_msg('exit!') +@dsl.pipeline(name='pipeline-with-exit-handler') +def pipeline_exit_handler(message: str = 'Hello World!'): + + exit_task = print_op(message='Exit handler has worked!') with dsl.ExitHandler(exit_task): - download_task = gcs_download_op(url) - echo_task = print_file(download_task.output) + print_op(message=message) + fail_op(message='Task failed.') if __name__ == '__main__': - compiler.Compiler().compile(pipeline_exit_handler, __file__ + '.yaml') + compiler.Compiler().compile( + pipeline_func=pipeline_exit_handler, package_path=__file__ + '.yaml') diff --git a/samples/core/exit_handler/exit_handler_test.py b/samples/core/exit_handler/exit_handler_test.py index 94b19f5380..5e801d4dc6 100644 --- a/samples/core/exit_handler/exit_handler_test.py +++ b/samples/core/exit_handler/exit_handler_test.py @@ -17,11 +17,10 @@ import unittest from pprint import pprint -import kfp.deprecated as kfp +import kfp as kfp import kfp_server_api -from .exit_handler import pipeline_exit_handler -from .exit_handler_v2 import pipeline_exit_handler as pipeline_exit_handler_v2 +from .exit_handler import pipeline_exit_handler as pipeline_exit_handler from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient @@ -53,10 +52,6 @@ def verify(mlmd_connection_config, run: kfp_server_api.ApiRun, **kwargs): run_pipeline_func([ TestCase( pipeline_func=pipeline_exit_handler, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), - TestCase( - pipeline_func=pipeline_exit_handler_v2, mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/core/kubernetes_pvc/README.md b/samples/core/kubernetes_pvc/README.md new file mode 100644 index 0000000000..8384b2b37f --- /dev/null +++ b/samples/core/kubernetes_pvc/README.md @@ -0,0 +1,12 @@ +## Read/write to a Kubernetes PVC using kfp-kubernetes + +This sample uses [kfp-kubernetes](https://pypi.org/project/kfp-kubernetes/) to +demonstrate typical usage of a plugin library. Specifically, we will use +`kfp-kubernetes` to create a [PersistentVolumeClaim +(PVC)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/), use the +PVC to pass data between tasks, and delete the PVC after using it. + +See the [kfp-kubernetes documentation](https://kfp-kubernetes.readthedocs.io/) +and [Kubeflow Pipeline +documentation](https://www.kubeflow.org/docs/components/pipelines/v2/platform-specific-features/#example-readwrite-to-a-kubernetes-pvc-using-kfp-kubernetes) +for more information. diff --git a/samples/core/kubernetes_pvc/kubernetes_pvc.py b/samples/core/kubernetes_pvc/kubernetes_pvc.py new file mode 100644 index 0000000000..1b82772f65 --- /dev/null +++ b/samples/core/kubernetes_pvc/kubernetes_pvc.py @@ -0,0 +1,62 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kfp import dsl, compiler +from kfp import kubernetes + +@dsl.component +def make_data(): + with open('/data/file.txt', 'w') as f: + f.write('my data') + +@dsl.component +def read_data(): + with open('/reused_data/file.txt') as f: + print(f.read()) + +@dsl.pipeline( + name="kubernetes-pvc-basic", + description="A Basic Example on Kubernetes PVC Usage." +) +def my_pipeline(): + pvc1 = kubernetes.CreatePVC( + # can also use pvc_name instead of pvc_name_suffix to use a pre-existing PVC + pvc_name_suffix='-my-pvc', + access_modes=['ReadWriteOnce'], + size='5Gi', + storage_class_name='standard', + ) + + task1 = make_data() + # normally task sequencing is handled by data exchange via component inputs/outputs + # but since data is exchanged via volume, we need to call .after explicitly to sequence tasks + task2 = read_data().after(task1) + + kubernetes.mount_pvc( + task1, + pvc_name=pvc1.outputs['name'], + mount_path='/data', + ) + kubernetes.mount_pvc( + task2, + pvc_name=pvc1.outputs['name'], + mount_path='/reused_data', + ) + + # wait to delete the PVC until after task2 completes + delete_pvc1 = kubernetes.DeletePVC( + pvc_name=pvc1.outputs['name']).after(task2) + +if __name__ == '__main__': + compiler.Compiler().compile(my_pipeline, __file__ + '.yaml') diff --git a/samples/core/loop_output/loop_output.py b/samples/core/loop_output/loop_output.py index 2ff75e5fa9..b6e6da4863 100644 --- a/samples/core/loop_output/loop_output.py +++ b/samples/core/loop_output/loop_output.py @@ -12,22 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.deprecated import components -from kfp.deprecated import dsl +from kfp import compiler, dsl -@components.create_component_from_func +@dsl.component def args_generator_op() -> str: return '[1.1, 1.2, 1.3]' -@components.create_component_from_func -def print_op(s: float): +# TODO(Bobgy): how can we make this component with type float? +# got error: kfp.components.types.type_utils.InconsistentTypeException: +# Incompatible argument passed to the input "s" of component "Print op": Argument +# type "STRING" is incompatible with the input type "NUMBER_DOUBLE" +@dsl.component +def print_op(s: str): print(s) -@dsl.pipeline(name='pipeline-with-loop-output') +@dsl.pipeline(name='pipeline-with-loop-output-v2') def my_pipeline(): args_generator = args_generator_op() with dsl.ParallelFor(args_generator.output) as item: - print_op(item) + print_op(s=item) + +if __name__ == '__main__': + compiler.Compiler().compile(my_pipeline, __file__ + '.yaml') diff --git a/samples/core/loop_output/loop_output_test.py b/samples/core/loop_output/loop_output_test.py index b9b49ae9ca..d2648cdc9f 100644 --- a/samples/core/loop_output/loop_output_test.py +++ b/samples/core/loop_output/loop_output_test.py @@ -15,11 +15,10 @@ from __future__ import annotations import unittest -import kfp.deprecated as kfp +import kfp import kfp_server_api -from ml_metadata.proto import Execution -from .loop_output import my_pipeline -from .loop_output_v2 import my_pipeline as my_pipeline_v2 +from ml_metadata.proto.metadata_store_pb2 import Execution +from loop_output import my_pipeline from kfp.samples.test.utils import KfpTask, run_pipeline_func, TestCase @@ -76,12 +75,8 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, run_pipeline_func([ TestCase( - pipeline_func=my_pipeline_v2, + pipeline_func=my_pipeline, mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, verify_func=verify, ), - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), ]) diff --git a/samples/core/loop_output/loop_output_v2.py b/samples/core/loop_output/loop_output_v2.py deleted file mode 100644 index c7dd4332d5..0000000000 --- a/samples/core/loop_output/loop_output_v2.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from kfp import dsl - -# In tests, we install a KFP package from the PR under test. Users should not -# normally need to specify `kfp_package_path` in their component definitions. -_KFP_PACKAGE_PATH = os.getenv('KFP_PACKAGE_PATH') - - -@dsl.component(kfp_package_path=_KFP_PACKAGE_PATH) -def args_generator_op() -> str: - return '[1.1, 1.2, 1.3]' - - -# TODO(Bobgy): how can we make this component with type float? -# got error: kfp.components.types.type_utils.InconsistentTypeException: -# Incompatible argument passed to the input "s" of component "Print op": Argument -# type "STRING" is incompatible with the input type "NUMBER_DOUBLE" -@dsl.component(kfp_package_path=_KFP_PACKAGE_PATH) -def print_op(s: str): - print(s) - - -@dsl.pipeline(name='pipeline-with-loop-output-v2') -def my_pipeline(): - args_generator = args_generator_op() - with dsl.ParallelFor(args_generator.output) as item: - print_op(s=item) diff --git a/samples/core/loop_parameter/loop_parameter.py b/samples/core/loop_parameter/loop_parameter.py index 1c5c786ed2..f171589601 100644 --- a/samples/core/loop_parameter/loop_parameter.py +++ b/samples/core/loop_parameter/loop_parameter.py @@ -1,31 +1,33 @@ -from kfp.deprecated import components, dsl -from typing import List +from kfp import compiler, dsl -@components.create_component_from_func +@dsl.component def print_op(text: str) -> str: print(text) return text -@components.create_component_from_func +@dsl.component def concat_op(a: str, b: str) -> str: print(a + b) return a + b -@components.create_component_from_func +@dsl.component def generate_op() -> str: import json return json.dumps([{'a': i, 'b': i * 10} for i in range(1, 5)]) @dsl.pipeline(name='pipeline-with-loop-parameter') -def my_pipeline(greeting:str='this is a test for looping through parameters'): +def my_pipeline( + greeting: str = 'this is a test for looping through parameters'): print_task = print_op(text=greeting) generate_task = generate_op() with dsl.ParallelFor(generate_task.output) as item: concat_task = concat_op(a=item.a, b=item.b) - concat_task.after(print_task) - print_task_2 = print_op(concat_task.output) + print_task_2 = print_op(text=concat_task.output) + +if __name__ == '__main__': + compiler.Compiler().compile(my_pipeline, __file__ + '.yaml') diff --git a/samples/core/loop_parameter/loop_parameter_test.py b/samples/core/loop_parameter/loop_parameter_test.py index 74c49622e4..648d96cb62 100644 --- a/samples/core/loop_parameter/loop_parameter_test.py +++ b/samples/core/loop_parameter/loop_parameter_test.py @@ -14,10 +14,9 @@ from __future__ import annotations import unittest -import kfp.deprecated as kfp +import kfp import kfp_server_api -from .loop_parameter import my_pipeline -from .loop_parameter_v2 import my_pipeline as my_pipeline_v2 +from loop_parameter import my_pipeline from kfp.samples.test.utils import KfpTask, debug_verify, run_pipeline_func, TestCase @@ -35,11 +34,7 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, run_pipeline_func([ TestCase( - pipeline_func=my_pipeline_v2, + pipeline_func=my_pipeline, mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, verify_func=verify), - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), ]) diff --git a/samples/core/loop_parameter/loop_parameter_v2.py b/samples/core/loop_parameter/loop_parameter_v2.py deleted file mode 100644 index 478f782056..0000000000 --- a/samples/core/loop_parameter/loop_parameter_v2.py +++ /dev/null @@ -1,36 +0,0 @@ -import os - -from kfp import dsl - -# In tests, we install a KFP package from the PR under test. Users should not -# normally need to specify `kfp_package_path` in their component definitions. -_KFP_PACKAGE_PATH = os.getenv('KFP_PACKAGE_PATH') - - -@dsl.component(kfp_package_path=_KFP_PACKAGE_PATH) -def print_op(text: str) -> str: - print(text) - return text - - -@dsl.component(kfp_package_path=_KFP_PACKAGE_PATH) -def concat_op(a: str, b: str) -> str: - print(a + b) - return a + b - - -@dsl.component(kfp_package_path=_KFP_PACKAGE_PATH) -def generate_op() -> str: - import json - return json.dumps([{'a': i, 'b': i * 10} for i in range(1, 5)]) - - -@dsl.pipeline(name='pipeline-with-loop-parameter') -def my_pipeline( - greeting: str = 'this is a test for looping through parameters'): - print_task = print_op(text=greeting) - - generate_task = generate_op() - with dsl.ParallelFor(generate_task.output) as item: - concat_task = concat_op(a=item.a, b=item.b) - print_task_2 = print_op(text=concat_task.output) diff --git a/samples/core/loop_static/loop_static.py b/samples/core/loop_static/loop_static.py index 202869338e..d19fe38915 100644 --- a/samples/core/loop_static/loop_static.py +++ b/samples/core/loop_static/loop_static.py @@ -1,30 +1,29 @@ -from kfp.deprecated import components, dsl -from typing import List +from kfp import compiler, dsl -@components.create_component_from_func +@dsl.component def print_op(text: str) -> str: print(text) return text -@components.create_component_from_func +@dsl.component def concat_op(a: str, b: str) -> str: print(a + b) return a + b -_DEFAULT_LOOP_ARGUMENTS = [{'a': '1', 'b': '2'}, {'a': '10', 'b': '20'}] - - @dsl.pipeline(name='pipeline-with-loop-static') def my_pipeline( - static_loop_arguments: List[dict] = _DEFAULT_LOOP_ARGUMENTS, - greeting: str = 'this is a test for looping through parameters', -): + greeting: str = 'this is a test for looping through parameters',): + import json print_task = print_op(text=greeting) + static_loop_arguments = [json.dumps({'a': '1', 'b': '2'}), + json.dumps({'a': '10', 'b': '20'})] with dsl.ParallelFor(static_loop_arguments) as item: concat_task = concat_op(a=item.a, b=item.b) - concat_task.after(print_task) - print_task_2 = print_op(concat_task.output) + print_task_2 = print_op(text=concat_task.output) + +if __name__ == '__main__': + compiler.Compiler().compile(my_pipeline, __file__ + '.yaml') diff --git a/samples/core/loop_static/loop_static_test.py b/samples/core/loop_static/loop_static_test.py index 909d6261f9..276e525997 100644 --- a/samples/core/loop_static/loop_static_test.py +++ b/samples/core/loop_static/loop_static_test.py @@ -14,10 +14,9 @@ from __future__ import annotations import unittest -import kfp.deprecated as kfp +import kfp import kfp_server_api -from .loop_static import my_pipeline -from .loop_static_v2 import my_pipeline as my_pipeline_v2 +from loop_static import my_pipeline from kfp.samples.test.utils import KfpTask, run_pipeline_func, TestCase @@ -50,12 +49,8 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, run_pipeline_func([ TestCase( - pipeline_func=my_pipeline_v2, + pipeline_func=my_pipeline, mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, verify_func=verify, ), - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), ]) diff --git a/samples/core/loop_static/loop_static_v2.py b/samples/core/loop_static/loop_static_v2.py deleted file mode 100644 index 2f634f77a0..0000000000 --- a/samples/core/loop_static/loop_static_v2.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -from typing import List - -from kfp import dsl - -# In tests, we install a KFP package from the PR under test. Users should not -# normally need to specify `kfp_package_path` in their component definitions. -_KFP_PACKAGE_PATH = os.getenv('KFP_PACKAGE_PATH') - - -@dsl.component(kfp_package_path=_KFP_PACKAGE_PATH) -def print_op(text: str) -> str: - print(text) - return text - - -@dsl.component(kfp_package_path=_KFP_PACKAGE_PATH) -def concat_op(a: str, b: str) -> str: - print(a + b) - return a + b - - -@dsl.pipeline(name='pipeline-with-loop-static') -def my_pipeline( - greeting: str = 'this is a test for looping through parameters',): - print_task = print_op(text=greeting) - static_loop_arguments = [{'a': '1', 'b': '2'}, {'a': '10', 'b': '20'}] - - with dsl.ParallelFor(static_loop_arguments) as item: - concat_task = concat_op(a=item.a, b=item.b) - print_task_2 = print_op(text=concat_task.output) diff --git a/samples/core/sequential/sequential.py b/samples/core/sequential/sequential.py index 39819c3dc1..db63a1fff2 100755 --- a/samples/core/sequential/sequential.py +++ b/samples/core/sequential/sequential.py @@ -17,35 +17,34 @@ from kfp import dsl, compiler -def gcs_download_op(url): - return dsl.ContainerOp( - name='GCS - Download', +@dsl.container_component +def gcs_download_op(url: str, output: dsl.OutputPath(str)): + return dsl.ContainerSpec( image='google/cloud-sdk:279.0.0', command=['sh', '-c'], - arguments=['gsutil cat $0 | tee $1', url, '/tmp/results.txt'], - file_outputs={ - 'data': '/tmp/results.txt', - } + args=['gsutil cat $0 | tee $1', url, output], ) -def echo_op(text): - return dsl.ContainerOp( - name='echo', +@dsl.container_component +def echo_op(text: str): + return dsl.ContainerSpec( image='library/bash:4.4.23', command=['sh', '-c'], - arguments=['echo "$0"', text] + args=['echo "$0"', text] ) + @dsl.pipeline( name='sequential-pipeline', description='A pipeline with two sequential steps.' ) -def sequential_pipeline(url='gs://ml-pipeline/sample-data/shakespeare/shakespeare1.txt'): +def sequential_pipeline(url: str = 'gs://ml-pipeline/sample-data/shakespeare/shakespeare1.txt'): """A pipeline with two sequential steps.""" - download_task = gcs_download_op(url) - echo_task = echo_op(download_task.output) + download_task = gcs_download_op(url=url) + echo_task = echo_op(text=download_task.output) + if __name__ == '__main__': compiler.Compiler().compile(sequential_pipeline, __file__ + '.yaml') diff --git a/samples/core/volume_ops/README.md b/samples/core/volume_ops/README.md deleted file mode 100644 index 7e07379046..0000000000 --- a/samples/core/volume_ops/README.md +++ /dev/null @@ -1,75 +0,0 @@ -## Simplify the creation of `PersistentVolumeClaim` instances - -**`VolumeOp`:** A specified `ResourceOp` for PVC creation. - -### Arguments: -The following arguments are an extension to the `ResourceOp` arguments. -If a `k8s_resource` is passed, then none of the following may be provided. - -* `resource_name`: The name of the resource which will be created. - This string will be prepended with the workflow name. - This may contain `PipelineParam`s. - (_required_) -* `size`: The requested size for the PVC. - This may contain `PipelineParam`s. - (_required_) -* `storage_class`: The storage class to be used. - This may contain `PipelineParam`s. - (_optional_) -* `modes`: The `accessModes` of the PVC. - Check - [this documentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) - for further information. - The user may find the following modes built-in: - * `VOLUME_MODE_RWO`: `["ReadWriteOnce"]` - * `VOLUME_MODE_RWM`: `["ReadWriteMany"]` - * `VOLUME_MODE_ROM`: `["ReadOnlyMany"]` - - Defaults to `VOLUME_MODE_RWM`. -* `annotations`: Annotations to be patched in the PVC. - These may contain `PipelineParam`s. - (_optional_) -* `data_source`: It is used to create a PVC from a `VolumeSnapshot`. - Can be either a `V1TypedLocalObjectReference` or a `string`, and may contain `PipelineParam`s. - (_Alpha feature_, _optional_) - -### Outputs -Additionally to the whole specification of the resource and its name (`ResourceOp` defaults), a -`VolumeOp` also outputs the storage size of the bounded PV (as `step.outputs["size"]`). -However, this may be empty if the storage provisioner has a `WaitForFirstConsumer` binding mode. -This value, if not empty, is always ≥ the requested size. - -### Useful attributes -1. The `VolumeOp` step has a `.volume` attribute which is a `PipelineVolume` referencing the - created PVC. - A `PipelineVolume` is essentially a `V1Volume` supplemented with an `.after()` method extending - the carried dependencies. - These dependencies can then be parsed properly by a `ContainerOp`, if used with the `pvolumes` - attribute, to extend the `ContainerOp`'s dependencies. -2. A `ContainerOp` has a `pvolumes` argument in its constructor. - This is a dictionary with mount paths as keys and volumes as values and functions similarly to - `file_outputs` (which can then be used as `op.outputs["key"]` or `op.output`). - For example: - ```python - vop = dsl.VolumeOp( - name="volume_creation", - resource_name="mypvc", - size="1Gi" - ) - step1 = dsl.ContainerOp( - name="step1", - ... - pvolumes={"/mnt": vop.volume} # Implies execution after vop - ) - step2 = dsl.ContainerOp( - name="step2", - ... - pvolumes={"/data": step1.pvolume, # Implies execution after step1 - "/mnt": dsl.PipelineVolume(pvc="existing-pvc")} - ) - step3 = dsl.ContainerOp( - name="step3", - ... - pvolumes={"/common": step2.pvolumes["/mnt"]} # Implies execution after step2 - ) - ``` diff --git a/samples/core/volume_ops/volume_ops.py b/samples/core/volume_ops/volume_ops.py deleted file mode 100644 index 5ce57f12cb..0000000000 --- a/samples/core/volume_ops/volume_ops.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2019-2023 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp import components, dsl, compiler - -@components.create_component_from_func -def write_to_volume(): - with open("/mnt/file.txt", "w") as file: - file.write("Hello world") - - -@dsl.pipeline( - name="volumeop-basic", - description="A Basic Example on VolumeOp Usage." -) -def volumeop_basic(size: str="1Gi"): - vop = dsl.VolumeOp( - name="create-pvc", - resource_name="my-pvc", - modes=dsl.VOLUME_MODE_RWO, - size=size - ) - - write_to_volume().add_pvolumes({"/mnt": vop.volume}) - - -if __name__ == '__main__': - compiler.Compiler().compile(volumeop_basic, __file__ + '.yaml') diff --git a/samples/test/after.py b/samples/test/after.py index 5c036c42b8..5a776a3910 100644 --- a/samples/test/after.py +++ b/samples/test/after.py @@ -42,6 +42,4 @@ def my_pipeline(): if __name__ == '__main__': compiler.Compiler().compile( - pipeline_func=my_pipeline, - pipeline_root='dummy_root', - output_path=__file__ + '.json') + pipeline_func=my_pipeline, output_path=__file__ + '.json') diff --git a/samples/v2/pipeline_with_env.py b/samples/v2/pipeline_with_env.py index c790615758..6204094470 100644 --- a/samples/v2/pipeline_with_env.py +++ b/samples/v2/pipeline_with_env.py @@ -47,7 +47,7 @@ def print_env_op(): """) -@dsl.pipeline(name='pipeline-with-env', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-env') def pipeline_with_env(): print_env_op().set_env_variable(name='ENV1', value='val1') print_env_2_op().set_env_variable( @@ -57,5 +57,4 @@ def pipeline_with_env(): if __name__ == '__main__': compiler.Compiler().compile( - pipeline_func=pipeline_with_env, - package_path='pipeline_with_env.yaml') \ No newline at end of file + pipeline_func=pipeline_with_env, package_path='pipeline_with_env.yaml') diff --git a/scripts/deploy/github/test-dynamic-loop.sh b/scripts/deploy/github/test-dynamic-loop.sh index 395592af18..125a5584aa 100755 --- a/scripts/deploy/github/test-dynamic-loop.sh +++ b/scripts/deploy/github/test-dynamic-loop.sh @@ -19,7 +19,7 @@ if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi source "${C_DIR}/test-pipeline.sh" RESULT=0 -run_test_case "dynamic-loop" "samples/core/loop_parameter/loop_parameter_v2.py" "SUCCEEDED" 20 || RESULT=$? +run_test_case "dynamic-loop" "samples/core/loop_parameter/loop_parameter.py" "SUCCEEDED" 20 || RESULT=$? STATUS_MSG=PASSED if [[ "$RESULT" -ne 0 ]]; then diff --git a/scripts/deploy/github/test-static-loop.sh b/scripts/deploy/github/test-static-loop.sh index f5e12400ec..84fea57e5a 100755 --- a/scripts/deploy/github/test-static-loop.sh +++ b/scripts/deploy/github/test-static-loop.sh @@ -19,7 +19,7 @@ if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi source "${C_DIR}/test-pipeline.sh" RESULT=0 -run_test_case "static-loop" "samples/core/loop_static/loop_static_v2.py" "SUCCEEDED" 20 || RESULT=$? +run_test_case "static-loop" "samples/core/loop_static/loop_static.py" "SUCCEEDED" 20 || RESULT=$? STATUS_MSG=PASSED if [[ "$RESULT" -ne 0 ]]; then diff --git a/sdk/RELEASE.md b/sdk/RELEASE.md index 00bda01c27..61dfc86d14 100644 --- a/sdk/RELEASE.md +++ b/sdk/RELEASE.md @@ -3,12 +3,36 @@ ## Features ## Breaking changes +* Support collecting outputs from conditional branches using `dsl.OneOf` [\#10067](https://github.com/kubeflow/pipelines/pull/10067) ## Deprecations +* Add notice of Python 3.7 support removal on April 23, 2024 [\#10139](https://github.com/kubeflow/pipelines/pull/10139) ## Bug fixes and other changes +* Fix type on `dsl.ParallelFor` sub-DAG output when a `dsl.Collected` is used. Non-functional fix. [\#10069](https://github.com/kubeflow/pipelines/pull/10069) +* Fix bug when `dsl.importer` argument is provided by a `dsl.ParallelFor` loop variable. [\#10116](https://github.com/kubeflow/pipelines/pull/10116) ## Documentation updates + +# 2.3.0 +## Features +* Support `PipelineTaskFinalStatus` in tasks that use `.ignore_upstream_failure()` [\#10010](https://github.com/kubeflow/pipelines/pull/10010) +* Add support for a Pythonic artifact authoring style [\#9932](https://github.com/kubeflow/pipelines/pull/9932) + + +## Breaking changes + +## Deprecations + +## Bug fixes and other changes + +## Documentation updates +# 2.2.0 + +## Features +* Add support for `dsl.If`, `dsl.Elif`, and `dsl.Else` control flow context managers; deprecate `dsl.Condition` in favor of `dsl.If` [\#9894](https://github.com/kubeflow/pipelines/pull/9894) +* Create "dependency-free" runtime package (only `typing_extensions` required) for Lightweight Python Components to reduce runtime dependency resolution errors [\#9710](https://github.com/kubeflow/pipelines/pull/9710), [\#9886](https://github.com/kubeflow/pipelines/pull/9886) + # 2.0.1 ## Features diff --git a/sdk/python/kfp/__init__.py b/sdk/python/kfp/__init__.py index d58174a9b1..eb1fce1d7e 100644 --- a/sdk/python/kfp/__init__.py +++ b/sdk/python/kfp/__init__.py @@ -16,9 +16,27 @@ # https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages __path__ = __import__('pkgutil').extend_path(__path__, __name__) -__version__ = '2.0.1' +__version__ = '2.3.0' + +import sys +import warnings + +if sys.version_info < (3, 8): + warnings.warn( + ('Python 3.7 has reached end-of-life. KFP will drop support for Python 3.7 on April 23, 2024. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3.8. See https://devguide.python.org/versions/ for more details.' + ), + FutureWarning, + stacklevel=2, + ) TYPE_CHECK = True -from kfp import dsl -from kfp.client import Client +import os + +# compile-time only dependencies +if os.environ.get('_KFP_RUNTIME', 'false') != 'true': + # make `from kfp import components` and `from kfp import dsl` valid; + # related to namespace packaging issue + from kfp import components # noqa: keep unused import + from kfp import dsl # noqa: keep unused import + from kfp.client import Client # noqa: keep unused import diff --git a/sdk/python/kfp/cli/compile_.py b/sdk/python/kfp/cli/compile_.py index d265ca6194..2bd3bab18c 100644 --- a/sdk/python/kfp/cli/compile_.py +++ b/sdk/python/kfp/cli/compile_.py @@ -22,8 +22,8 @@ import click from kfp import compiler -from kfp.components import base_component -from kfp.components import graph_component +from kfp.dsl import base_component +from kfp.dsl import graph_component def is_pipeline_func(func: Callable) -> bool: diff --git a/sdk/python/kfp/cli/component.py b/sdk/python/kfp/cli/component.py index e4b8d7e321..079c200fe3 100644 --- a/sdk/python/kfp/cli/component.py +++ b/sdk/python/kfp/cli/component.py @@ -31,16 +31,14 @@ _DOCKER_IS_PRESENT = False import kfp as kfp -from kfp.components import component_factory -from kfp.components import kfp_config -from kfp.components import utils +from kfp.dsl import component_factory +from kfp.dsl import kfp_config +from kfp.dsl import utils _REQUIREMENTS_TXT = 'runtime-requirements.txt' _DOCKERFILE = 'Dockerfile' -# TODO: merge kfp_package_path into runtime-requirements.txt, once we have -# kfp_runtime package that is dependency-free. _DOCKERFILE_TEMPLATE = ''' FROM {base_image} diff --git a/sdk/python/kfp/cli/component_test.py b/sdk/python/kfp/cli/component_test.py index 4d256afaf2..a818aafb80 100644 --- a/sdk/python/kfp/cli/component_test.py +++ b/sdk/python/kfp/cli/component_test.py @@ -86,6 +86,7 @@ def setUp(self) -> None: }] self._docker_client.images.push.return_value = [{'status': 'Pushed'}] self.addCleanup(patcher.stop) + self.current_dir = os.path.dirname(os.path.abspath(__file__)) with contextlib.ExitStack() as stack: stack.enter_context(self.runner.isolated_filesystem()) @@ -579,8 +580,7 @@ def test_dockerfile_can_contain_custom_kfp_package(self): component = _make_component( func_name='train', target_image='custom-image') _write_components('components.py', component) - current_dir = os.path.dirname(os.path.abspath(__file__)) - package_dir = os.path.dirname(os.path.dirname(current_dir)) + package_dir = os.path.dirname(os.path.dirname(self.current_dir)) # suppresses large stdout from subprocess that builds kfp package with mock.patch.object( diff --git a/sdk/python/kfp/cli/utils/aliased_plurals_group.py b/sdk/python/kfp/cli/utils/aliased_plurals_group.py index 79631827de..79c152868a 100644 --- a/sdk/python/kfp/cli/utils/aliased_plurals_group.py +++ b/sdk/python/kfp/cli/utils/aliased_plurals_group.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple, Union +from typing import List, Optional, Tuple import click @@ -30,8 +30,8 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command: raise click.UsageError(f"Unrecognized command '{cmd_name}'") def resolve_command( - self, ctx: click.Context, args: List[str] - ) -> Tuple[Union[str, None], Union[click.Command, None], List[str]]: + self, ctx: click.Context, + args: List[str]) -> Tuple[Optional[str], Optional[None], List[str]]: # always return the full command name _, cmd, args = super().resolve_command(ctx, args) return cmd.name, cmd, args # type: ignore diff --git a/sdk/python/kfp/cli/utils/deprecated_alias_group.py b/sdk/python/kfp/cli/utils/deprecated_alias_group.py index b4720ab92f..307fdabb49 100644 --- a/sdk/python/kfp/cli/utils/deprecated_alias_group.py +++ b/sdk/python/kfp/cli/utils/deprecated_alias_group.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Optional, Tuple import click @@ -58,7 +58,7 @@ def get_command(self, ctx: click.Context, def resolve_command( self, ctx: click.Context, args: List[str] - ) -> Tuple[Union[str, None], Union[click.Command, None], List[str]]: + ) -> Tuple[Optional[str], Optional[click.Command], List[str]]: # always return the full command name _, cmd, args = super().resolve_command(ctx, args) return cmd.name, cmd, args # type: ignore diff --git a/sdk/python/kfp/client/auth.py b/sdk/python/kfp/client/auth.py index 75b0faef48..8a68183345 100644 --- a/sdk/python/kfp/client/auth.py +++ b/sdk/python/kfp/client/auth.py @@ -16,7 +16,7 @@ import json import logging import os -from typing import Any, Callable, Dict, Generator, Iterable, Tuple, Union +from typing import Any, Callable, Dict, Generator, Iterable, Optional, Tuple from urllib.parse import parse_qs from urllib.parse import urlparse from webbrowser import open_new_tab @@ -38,7 +38,7 @@ LOCAL_KFP_CREDENTIAL = os.path.expanduser('~/.config/kfp/credentials.json') -def get_gcp_access_token() -> Union[str, None]: +def get_gcp_access_token() -> Optional[str]: """Gets GCP access token for the current Application Default Credentials. Returns: @@ -60,7 +60,7 @@ def get_gcp_access_token() -> Union[str, None]: def get_auth_token(client_id: str, other_client_id: str, - other_client_secret: str) -> Tuple[Union[str, None], bool]: + other_client_secret: str) -> Tuple[Optional[str], bool]: """Gets auth token from default service account or user account. Returns: @@ -122,7 +122,7 @@ def get_auth_token(client_id: str, other_client_id: str, return token, is_refresh_token -def get_auth_token_from_sa(client_id: str) -> Union[str, None]: +def get_auth_token_from_sa(client_id: str) -> Optional[str]: """Gets auth token from default service account. Returns: @@ -135,8 +135,7 @@ def get_auth_token_from_sa(client_id: str) -> Union[str, None]: def get_service_account_credentials( - client_id: str -) -> Union[google.oauth2.service_account.Credentials, None]: + client_id: str) -> Optional[google.oauth2.service_account.Credentials]: """Figure out what environment we're running in and get some preliminary information about the service account. @@ -262,11 +261,13 @@ def get_auth_code(client_id: str) -> Tuple[str, str]: 'scope=openid%20email&access_type=offline&' f'redirect_uri={redirect_uri}') authorization_response = None - if ('SSH_CONNECTION' in os.environ) or ('SSH_CLIENT' in os.environ): + if ('SSH_CONNECTION' in os.environ) or ('SSH_CLIENT' + in os.environ) or is_ipython(): try: - print(( - 'SSH connection detected. Please follow the instructions below.' - ' Otherwise, press CTRL+C if you are not connected via SSH.')) + print(('SSH connection or IPython shell detected. Please follow the' + ' instructions below. Otherwise, press CTRL+C if you are not' + ' connected via SSH and not using IPython (e.g. Jupyter' + ' Notebook).')) authorization_response = get_auth_response_ssh(host, port, auth_url) except KeyboardInterrupt: authorization_response = None @@ -318,7 +319,7 @@ def get_auth_response_ssh(host: str, port: int, auth_url: str) -> str: def get_auth_response_local(host: str, port: int, - auth_url: str) -> Union[str, None]: + auth_url: str) -> Optional[str]: """Fetches OAuth authorization response URL using a local web-server. Args: @@ -510,3 +511,15 @@ def fetch_auth_token_from_response(url: str) -> str: if isinstance(access_code, list): access_code = str(access_code.pop(0)) return access_code + + +def is_ipython() -> bool: + """Returns whether we are running in notebook.""" + try: + import IPython + ipy = IPython.get_ipython() + if ipy is None: + return False + except ImportError: + return False + return True diff --git a/sdk/python/kfp/client/auth_test.py b/sdk/python/kfp/client/auth_test.py new file mode 100644 index 0000000000..db58b51adb --- /dev/null +++ b/sdk/python/kfp/client/auth_test.py @@ -0,0 +1,100 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest.mock import MagicMock +from unittest.mock import patch + +from absl.testing import parameterized +from kfp.client import auth + + +class TestAuth(parameterized.TestCase): + + def test_is_ipython_return_false(self): + mock = MagicMock() + with patch.dict('sys.modules', IPython=mock): + mock.get_ipython.return_value = None + self.assertFalse(auth.is_ipython()) + + def test_is_ipython_return_true(self): + mock = MagicMock() + with patch.dict('sys.modules', IPython=mock): + mock.get_ipython.return_value = 'Something' + self.assertTrue(auth.is_ipython()) + + def test_is_ipython_should_raise_error(self): + mock = MagicMock() + with patch.dict('sys.modules', mock): + mock.side_effect = ImportError + self.assertFalse(auth.is_ipython()) + + @patch('builtins.input', lambda *args: + 'https://oauth2.example.com/auth?code=4/P7q7W91a-oMsCeLvIaQm6bTrgtp7' + ) + @patch('kfp.client.auth.is_ipython', lambda *args: True) + @patch.dict(os.environ, dict(), clear=True) + def test_get_auth_code_from_ipython(self): + token, redirect_uri = auth.get_auth_code('sample-client-id') + self.assertEqual(token, '4/P7q7W91a-oMsCeLvIaQm6bTrgtp7') + self.assertEqual(redirect_uri, 'http://localhost:9901') + + @patch('builtins.input', lambda *args: + 'https://oauth2.example.com/auth?code=4/P7q7W91a-oMsCeLvIaQm6bTrgtp7' + ) + @patch('kfp.client.auth.is_ipython', lambda *args: False) + @patch.dict(os.environ, {'SSH_CONNECTION': 'ENABLED'}, clear=True) + def test_get_auth_code_from_remote_connection(self): + token, redirect_uri = auth.get_auth_code('sample-client-id') + self.assertEqual(token, '4/P7q7W91a-oMsCeLvIaQm6bTrgtp7') + self.assertEqual(redirect_uri, 'http://localhost:9901') + + @patch('builtins.input', lambda *args: + 'https://oauth2.example.com/auth?code=4/P7q7W91a-oMsCeLvIaQm6bTrgtp7' + ) + @patch('kfp.client.auth.is_ipython', lambda *args: False) + @patch.dict(os.environ, {'SSH_CLIENT': 'ENABLED'}, clear=True) + def test_get_auth_code_from_remote_client(self): + token, redirect_uri = auth.get_auth_code('sample-client-id') + self.assertEqual(token, '4/P7q7W91a-oMsCeLvIaQm6bTrgtp7') + self.assertEqual(redirect_uri, 'http://localhost:9901') + + @patch('builtins.input', lambda *args: 'https://oauth2.example.com/auth') + @patch('kfp.client.auth.is_ipython', lambda *args: False) + @patch.dict(os.environ, {'SSH_CLIENT': 'ENABLED'}, clear=True) + def test_get_auth_code_from_remote_client_missing_code(self): + self.assertRaises(KeyError, auth.get_auth_code, 'sample-client-id') + + @patch('kfp.client.auth.get_auth_response_local', lambda *args: + 'https://oauth2.example.com/auth?code=4/P7q7W91a-oMsCeLvIaQm6bTrgtp7' + ) + @patch('kfp.client.auth.is_ipython', lambda *args: False) + @patch.dict(os.environ, dict(), clear=True) + def test_get_auth_code_from_local(self): + token, redirect_uri = auth.get_auth_code('sample-client-id') + self.assertEqual(token, '4/P7q7W91a-oMsCeLvIaQm6bTrgtp7') + self.assertEqual(redirect_uri, 'http://localhost:9901') + + @patch('kfp.client.auth.get_auth_response_local', lambda *args: None) + @patch('kfp.client.auth.is_ipython', lambda *args: False) + @patch.dict(os.environ, dict(), clear=True) + def test_get_auth_code_from_local_empty_response(self): + self.assertRaises(ValueError, auth.get_auth_code, 'sample-client-id') + + @patch('kfp.client.auth.get_auth_response_local', + lambda *args: 'this-is-an-invalid-response') + @patch('kfp.client.auth.is_ipython', lambda *args: False) + @patch.dict(os.environ, dict(), clear=True) + def test_get_auth_code_from_local_invalid_response(self): + self.assertRaises(KeyError, auth.get_auth_code, 'sample-client-id') diff --git a/sdk/python/kfp/client/client.py b/sdk/python/kfp/client/client.py index 8127b1b232..bdf9cbdf20 100644 --- a/sdk/python/kfp/client/client.py +++ b/sdk/python/kfp/client/client.py @@ -32,7 +32,8 @@ from kfp import compiler from kfp.client import auth from kfp.client import set_volume_credentials -from kfp.components import base_component +from kfp.client.token_credentials_base import TokenCredentialsBase +from kfp.dsl import base_component from kfp.pipeline_spec import pipeline_spec_pb2 import kfp_server_api import yaml @@ -150,7 +151,7 @@ def __init__( proxy: Optional[str] = None, ssl_ca_cert: Optional[str] = None, kube_context: Optional[str] = None, - credentials: Optional[str] = None, + credentials: Optional[TokenCredentialsBase] = None, ui_host: Optional[str] = None, verify_ssl: Optional[bool] = None, ) -> None: @@ -221,7 +222,7 @@ def _load_config( proxy: Optional[str], ssl_ca_cert: Optional[str], kube_context: Optional[str], - credentials: Optional[str], + credentials: Optional[TokenCredentialsBase], verify_ssl: Optional[bool], ) -> kfp_server_api.Configuration: config = kfp_server_api.Configuration() @@ -324,18 +325,6 @@ def _load_config( def _is_inverse_proxy_host(self, host: str) -> bool: return bool(re.match(r'\S+.googleusercontent.com/{0,1}$', host)) - def _is_ipython(self) -> bool: - """Returns whether we are running in notebook.""" - try: - import IPython - ipy = IPython.get_ipython() - if ipy is None: - return False - except ImportError: - return False - - return True - def _get_url_prefix(self) -> str: if self._uihost: # User's own connection. @@ -488,7 +477,7 @@ def create_experiment( experiment = self._experiment_api.create_experiment(body=experiment) link = f'{self._get_url_prefix()}/#/experiments/details/{experiment.experiment_id}' - if self._is_ipython(): + if auth.is_ipython(): import IPython html = f'Experiment details.' IPython.display.display(IPython.display.HTML(html)) @@ -744,7 +733,7 @@ def run_pipeline( response = self._run_api.create_run(body=run_body) link = f'{self._get_url_prefix()}/#/runs/details/{response.run_id}' - if self._is_ipython(): + if auth.is_ipython(): import IPython html = (f'Run details.') IPython.display.display(IPython.display.HTML(html)) @@ -1424,7 +1413,7 @@ def upload_pipeline( description=description, namespace=namespace) link = f'{self._get_url_prefix()}/#/pipelines/details/{response.pipeline_id}' - if self._is_ipython(): + if auth.is_ipython(): import IPython html = f'Pipeline details.' IPython.display.display(IPython.display.HTML(html)) @@ -1473,7 +1462,7 @@ def upload_pipeline_version( pipeline_package_path, **kwargs) link = f'{self._get_url_prefix()}/#/pipelines/details/{response.pipeline_id}/version/{response.pipeline_version_id}' - if self._is_ipython(): + if auth.is_ipython(): import IPython html = f'Pipeline details.' IPython.display.display(IPython.display.HTML(html)) diff --git a/sdk/python/kfp/client/client_test.py b/sdk/python/kfp/client/client_test.py index 03a6632c0c..da6b0710b9 100644 --- a/sdk/python/kfp/client/client_test.py +++ b/sdk/python/kfp/client/client_test.py @@ -17,12 +17,12 @@ import tempfile import textwrap import unittest -from unittest.mock import MagicMock from unittest.mock import Mock from unittest.mock import patch from absl.testing import parameterized from google.protobuf import json_format +from kfp.client import auth from kfp.client import client from kfp.compiler import Compiler from kfp.dsl import component @@ -194,24 +194,6 @@ class TestClient(parameterized.TestCase): def setUp(self): self.client = client.Client(namespace='ns1') - def test__is_ipython_return_false(self): - mock = MagicMock() - with patch.dict('sys.modules', IPython=mock): - mock.get_ipython.return_value = None - self.assertFalse(self.client._is_ipython()) - - def test__is_ipython_return_true(self): - mock = MagicMock() - with patch.dict('sys.modules', IPython=mock): - mock.get_ipython.return_value = 'Something' - self.assertTrue(self.client._is_ipython()) - - def test__is_ipython_should_raise_error(self): - mock = MagicMock() - with patch.dict('sys.modules', mock): - mock.side_effect = ImportError - self.assertFalse(self.client._is_ipython()) - def test_wait_for_run_completion_invalid_token_should_raise_error(self): with self.assertRaises(kfp_server_api.ApiException): with patch.object( @@ -371,7 +353,7 @@ def pipeline_test_upload_without_name(boolean: bool = True): with patch.object(self.client._upload_api, 'upload_pipeline') as mock_upload_pipeline: - with patch.object(self.client, '_is_ipython', return_value=False): + with patch.object(auth, 'is_ipython', return_value=False): with tempfile.TemporaryDirectory() as tmp_path: pipeline_test_path = os.path.join(tmp_path, 'test.yaml') Compiler().compile( @@ -401,7 +383,7 @@ def pipeline_test_upload_without_name(boolean: bool = True): def test_upload_pipeline_with_name(self, pipeline_name): with patch.object(self.client._upload_api, 'upload_pipeline') as mock_upload_pipeline: - with patch.object(self.client, '_is_ipython', return_value=False): + with patch.object(auth, 'is_ipython', return_value=False): self.client.upload_pipeline( pipeline_package_path='fake.yaml', pipeline_name=pipeline_name, @@ -421,7 +403,7 @@ def test_upload_pipeline_with_name(self, pipeline_name): def test_upload_pipeline_with_name_invalid(self, pipeline_name): with patch.object(self.client._upload_api, 'upload_pipeline') as mock_upload_pipeline: - with patch.object(self.client, '_is_ipython', return_value=False): + with patch.object(auth, 'is_ipython', return_value=False): with self.assertRaisesRegex( ValueError, 'Invalid pipeline name. Pipeline name cannot be empty or contain only whitespace.' diff --git a/sdk/python/kfp/compiler/compiler.py b/sdk/python/kfp/compiler/compiler.py index af762ea2b6..a77f606e89 100644 --- a/sdk/python/kfp/compiler/compiler.py +++ b/sdk/python/kfp/compiler/compiler.py @@ -20,8 +20,8 @@ from typing import Any, Dict, Optional from kfp.compiler import pipeline_spec_builder as builder -from kfp.components import base_component -from kfp.components.types import type_utils +from kfp.dsl import base_component +from kfp.dsl.types import type_utils class Compiler: diff --git a/sdk/python/kfp/compiler/compiler_test.py b/sdk/python/kfp/compiler/compiler_test.py index a9d5f97c16..049975f1c0 100644 --- a/sdk/python/kfp/compiler/compiler_test.py +++ b/sdk/python/kfp/compiler/compiler_test.py @@ -31,17 +31,19 @@ from kfp.cli import cli from kfp.compiler import compiler from kfp.compiler import compiler_utils -from kfp.components import graph_component -from kfp.components import pipeline_task -from kfp.components import yaml_component -from kfp.components.types import type_utils from kfp.dsl import Artifact from kfp.dsl import ContainerSpec +from kfp.dsl import Dataset +from kfp.dsl import graph_component from kfp.dsl import Input from kfp.dsl import Model from kfp.dsl import Output from kfp.dsl import OutputPath +from kfp.dsl import pipeline_task from kfp.dsl import PipelineTaskFinalStatus +from kfp.dsl import tasks_group +from kfp.dsl import yaml_component +from kfp.dsl.types import type_utils from kfp.pipeline_spec import pipeline_spec_pb2 import yaml @@ -61,18 +63,131 @@ - {outputPath: output_value} """) +### components used throughout tests ### + + +@dsl.component +def flip_coin() -> str: + import random + return 'heads' if random.randint(0, 1) == 0 else 'tails' + + +@dsl.component +def print_and_return(text: str) -> str: + print(text) + return text + + +@dsl.component +def roll_three_sided_die() -> str: + import random + val = random.randint(0, 2) + + if val == 0: + return 'heads' + elif val == 1: + return 'tails' + else: + return 'draw' + + +@dsl.component +def int_zero_through_three() -> int: + import random + return random.randint(0, 3) + + +@dsl.component +def print_op(message: str): + print(message) + + +@dsl.component +def producer_op() -> str: + return 'a' + + +@dsl.component +def dummy_op(msg: str = ''): + pass + + +@dsl.component +def hello_world(text: str) -> str: + """Hello world component.""" + return text + + +@dsl.component +def add(nums: List[int]) -> int: + return sum(nums) + + +@dsl.component +def comp(): + pass + + +@dsl.component +def return_1() -> int: + return 1 + + +@dsl.component +def args_generator_op() -> List[Dict[str, str]]: + return [{'A_a': '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}] + + +@dsl.component +def my_comp(string: str, model: bool) -> str: + return string + + +@dsl.component +def print_hello(): + print('hello') + + +@dsl.component +def cleanup(): + print('cleanup') + + +@dsl.component +def double(num: int) -> int: + return 2 * num + + +@dsl.component +def print_and_return_as_artifact(text: str, a: Output[Artifact]): + print(text) + with open(a.path, 'w') as f: + f.write(text) + + +@dsl.component +def print_and_return_with_output_key(text: str, output_key: OutputPath(str)): + print(text) + with open(output_key, 'w') as f: + f.write(text) + + +@dsl.component +def print_artifact(a: Input[Artifact]): + with open(a.path) as f: + print(f.read()) + + +########### + class TestCompilePipeline(parameterized.TestCase): def test_can_use_dsl_attribute_on_kfp(self): - @kfp.dsl.component - def identity(string: str) -> str: - return string - @kfp.dsl.pipeline def my_pipeline(string: str = 'string'): - op1 = identity(string=string) + op1 = print_and_return(text=string) with tempfile.TemporaryDirectory() as tmpdir: compiler.Compiler().compile( @@ -152,18 +267,6 @@ def simple_pipeline(): with open(target_json_file, 'r') as f: f.read() - def test_compile_pipeline_with_dsl_graph_component_should_raise_error(self): - - with self.assertRaisesRegex( - AttributeError, - "module 'kfp.dsl' has no attribute 'graph_component'"): - - @dsl.graph_component - def flip_coin_graph_component(): - flip = flip_coin_op() - with dsl.Condition(flip.output == 'heads'): - flip_coin_graph_component() - def test_compile_pipeline_with_misused_inputvalue_should_raise_error(self): upstream_op = components.load_component_from_text(""" @@ -192,7 +295,7 @@ def test_compile_pipeline_with_misused_inputvalue_should_raise_error(self): ' type "system.Model@0.0.1" cannot be paired with InputValuePlaceholder.' ): - @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') + @dsl.pipeline(name='test-pipeline') def my_pipeline(): downstream_op(model=upstream_op().output) @@ -213,7 +316,7 @@ def test_compile_pipeline_with_misused_inputpath_should_raise_error(self): TypeError, ' type "String" cannot be paired with InputPathPlaceholder.'): - @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') + @dsl.pipeline(name='test-pipeline') def my_pipeline(text: str): component_op(text=text) @@ -222,7 +325,7 @@ def test_compile_pipeline_with_missing_task_should_raise_error(self): with self.assertRaisesRegex(ValueError, 'Task is missing from pipeline.'): - @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') + @dsl.pipeline(name='test-pipeline') def my_pipeline(text: str): pass @@ -243,7 +346,7 @@ def test_compile_pipeline_with_misused_inputuri_should_raise_error(self): TypeError, ' type "Float" cannot be paired with InputUriPlaceholder.'): - @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') + @dsl.pipeline(name='test-pipeline') def my_pipeline(value: float): component_op(value=value) @@ -264,7 +367,7 @@ def test_compile_pipeline_with_misused_outputuri_should_raise_error(self): TypeError, ' type "Integer" cannot be paired with OutputUriPlaceholder.'): - @dsl.pipeline(name='test-pipeline', pipeline_root='dummy_root') + @dsl.pipeline(name='test-pipeline') def my_pipeline(): component_op() @@ -503,14 +606,6 @@ def my_pipeline(): def test_invalid_data_dependency_loop(self): - @dsl.component - def producer_op() -> str: - return 'a' - - @dsl.component - def dummy_op(msg: str = ''): - pass - with self.assertRaisesRegex( compiler_utils.InvalidTopologyException, r'Illegal task dependency across DSL context managers\. A downstream task cannot depend on an upstream task within a dsl\.ParallelFor context unless the downstream is within that context too or the outputs are begin fanned-in to a list using dsl\.Collected\. Found task dummy-op which depends on upstream task producer-op within an uncommon dsl\.ParallelFor context\.' @@ -525,14 +620,6 @@ def my_pipeline(val: bool): def test_invalid_data_dependency_condition(self): - @dsl.component - def producer_op() -> str: - return 'a' - - @dsl.component - def dummy_op(msg: str = ''): - pass - with self.assertRaisesRegex( compiler_utils.InvalidTopologyException, r'Illegal task dependency across DSL context managers\. A downstream task cannot depend on an upstream task within a dsl\.Condition context unless the downstream is within that context too\. Found task dummy-op which depends on upstream task producer-op within an uncommon dsl\.Condition context\.' @@ -547,14 +634,6 @@ def my_pipeline(val: bool): def test_valid_data_dependency_condition(self): - @dsl.component - def producer_op() -> str: - return 'a' - - @dsl.component - def dummy_op(msg: str = ''): - pass - @dsl.pipeline(name='test-pipeline') def my_pipeline(val: bool): with dsl.Condition(val == False): @@ -568,14 +647,6 @@ def my_pipeline(val: bool): def test_invalid_data_dependency_exit_handler(self): - @dsl.component - def producer_op() -> str: - return 'a' - - @dsl.component - def dummy_op(msg: str = ''): - pass - with self.assertRaisesRegex( compiler_utils.InvalidTopologyException, r'Illegal task dependency across DSL context managers\. A downstream task cannot depend on an upstream task within a dsl\.ExitHandler context unless the downstream is within that context too\. Found task dummy-op which depends on upstream task producer-op-2 within an uncommon dsl\.ExitHandler context\.' @@ -591,14 +662,6 @@ def my_pipeline(val: bool): def test_valid_data_dependency_exit_handler(self): - @dsl.component - def producer_op() -> str: - return 'a' - - @dsl.component - def dummy_op(msg: str = ''): - pass - @dsl.pipeline(name='test-pipeline') def my_pipeline(val: bool): first_producer = producer_op() @@ -650,10 +713,6 @@ def my_pipeline(text: bool): def test_task_final_status_parameter_type_is_used(self): # previously compiled to STRUCT type, so checking that this is updated - @dsl.component - def identity(string: str) -> str: - return string - @dsl.component def exit_comp(status: dsl.PipelineTaskFinalStatus): print(status) @@ -662,7 +721,7 @@ def exit_comp(status: dsl.PipelineTaskFinalStatus): def my_pipeline(): exit_task = exit_comp() with dsl.ExitHandler(exit_task=exit_task): - identity(string='hi') + print_and_return(text='hi') self.assertEqual( my_pipeline.pipeline_spec.components['comp-exit-comp'] @@ -733,13 +792,9 @@ def my_pipeline(text: bool): def test_pipeline_in_pipeline(self): - @dsl.component - def print_op(msg: str): - print(msg) - @dsl.pipeline(name='graph-component') def graph_component(msg: str): - print_op(msg=msg) + print_op(message=msg) @dsl.pipeline(name='test-pipeline') def my_pipeline(): @@ -766,43 +821,31 @@ def test_pipeline_with_invalid_output(self): with self.assertRaisesRegex( ValueError, r'Pipeline or component output not defined: msg1'): - @dsl.component - def print_op(msg: str) -> str: - print(msg) - @dsl.pipeline def my_pipeline() -> NamedTuple('Outputs', [ ('msg', str), ]): - task = print_op(msg='Hello') + task = print_and_return(text='Hello') output = collections.namedtuple('Outputs', ['msg1']) return output(task.output) def test_pipeline_with_missing_output(self): with self.assertRaisesRegex(ValueError, 'Missing pipeline output: msg'): - @dsl.component - def print_op(msg: str) -> str: - print(msg) - @dsl.pipeline def my_pipeline() -> NamedTuple('Outputs', [ ('msg', str), ]): - task = print_op(msg='Hello') + task = print_and_return(text='Hello') with self.assertRaisesRegex(ValueError, 'Missing pipeline output: model'): - @dsl.component - def print_op(msg: str) -> str: - print(msg) - @dsl.pipeline def my_pipeline() -> NamedTuple('Outputs', [ ('model', dsl.Model), ]): - task = print_op(msg='Hello') + task = print_and_return(text='Hello') class V2NamespaceAliasTest(unittest.TestCase): @@ -841,11 +884,6 @@ def test_import_modules(self): from kfp.v2 import compiler from kfp.v2 import dsl - @dsl.component - def hello_world(text: str) -> str: - """Hello world component.""" - return text - @dsl.pipeline(name='hello-world', description='A simple intro pipeline') def pipeline_hello_world(text: str = 'hi there'): """Hello world pipeline.""" @@ -892,10 +930,6 @@ class TestWriteToFileTypes(parameterized.TestCase): def make_pipeline_spec(self): - @dsl.component - def dummy_op(): - pass - @dsl.pipeline(name=self.pipeline_name) def my_pipeline(): task = dummy_op() @@ -990,11 +1024,6 @@ class TestCompileComponent(parameterized.TestCase): @parameterized.parameters(['.json', '.yaml', '.yml']) def test_compile_component_simple(self, extension: str): - @dsl.component - def hello_world(text: str) -> str: - """Hello world component.""" - return text - with tempfile.TemporaryDirectory() as tempdir: output_json = os.path.join(tempdir, f'component{extension}') compiler.Compiler().compile( @@ -1062,11 +1091,6 @@ def hello_world(text: str = 'default_string') -> str: def test_compile_component_with_pipeline_parameters_override(self): - @dsl.component - def hello_world(text: str) -> str: - """Hello world component.""" - return text - with tempfile.TemporaryDirectory() as tempdir: output_json = os.path.join(tempdir, 'component.yaml') compiler.Compiler().compile( @@ -1333,11 +1357,6 @@ class TestSetRetryCompilation(unittest.TestCase): def test_set_retry(self): - @dsl.component - def hello_world(text: str) -> str: - """Hello world component.""" - return text - @dsl.pipeline(name='hello-world', description='A simple intro pipeline') def pipeline_hello_world(text: str = 'hi there'): """Hello world pipeline.""" @@ -1369,10 +1388,6 @@ class TestMultipleExitHandlerCompilation(unittest.TestCase): def test_basic(self): - @dsl.component - def print_op(message: str): - print(message) - @dsl.pipeline(name='pipeline-with-multiple-exit-handlers') def my_pipeline(): first_exit_task = print_op(message='First exit task.') @@ -1415,10 +1430,6 @@ def my_pipeline(): def test_nested_unsupported(self): - @dsl.component - def print_op(message: str): - print(message) - with self.assertRaisesRegex( ValueError, r'ExitHandler can only be used within the outermost scope of a pipeline function definition\.' @@ -1591,22 +1602,6 @@ def my_pipeline(): .runtime_value.constant.bool_value) -# helper component defintions for the ValidLegalTopologies tests -@dsl.component -def print_op(message: str): - print(message) - - -@dsl.component -def return_1() -> int: - return 1 - - -@dsl.component -def args_generator_op() -> List[Dict[str, str]]: - return [{'A_a': '1', 'B_b': '2'}, {'A_a': '10', 'B_b': '20'}] - - class TestValidLegalTopologies(unittest.TestCase): def test_inside_of_root_group_permitted(self): @@ -1828,16 +1823,12 @@ def test_inner_parallelfor_can_iter_over_upstream_output(self): def str_to_list(string: str) -> List: return [string] - @dsl.component - def identity(string: str) -> str: - return string - @dsl.pipeline def my_pipeline(): with dsl.ParallelFor(['a', 'b', 'c']) as itema: t1 = str_to_list(string=itema) with dsl.ParallelFor(t1.output) as itemb: - identity(string=itemb) + print_and_return(text=itemb) with tempfile.TemporaryDirectory() as tempdir: package_path = os.path.join(tempdir, 'pipeline.yaml') @@ -1850,10 +1841,6 @@ def test_permitted_nested_parallelfor_complex(self): def str_to_list(string: str) -> List: return [string] - @dsl.component - def identity(string: str) -> str: - return string - @dsl.pipeline def my_pipeline(): @@ -1862,18 +1849,18 @@ def my_pipeline(): t1 = str_to_list(string=itema) t2 = str_to_list(string=itema) - sequential_task1 = identity(string=itema) - identity(string=sequential_task1.output) + sequential_task1 = print_and_return(text=itema) + print_and_return(text=sequential_task1.output) # for-loop-3 with dsl.ParallelFor(t1.output) as itemb: t3 = str_to_list(string=itema) with dsl.ParallelFor(t3.output) as itemc: - identity(string=itemc) + print_and_return(text=itemc) with dsl.ParallelFor(t2.output) as itemd: - identity(string=itemd) + print_and_return(text=itemd) with dsl.ParallelFor(t2.output) as iteme: - identity(string=iteme) + print_and_return(text=iteme) with tempfile.TemporaryDirectory() as tempdir: package_path = os.path.join(tempdir, 'pipeline.yaml') @@ -1937,10 +1924,6 @@ def test_inner_task_prevented(self): r'Illegal task dependency across DSL context managers\. A downstream task cannot depend on an upstream task within a dsl\.ExitHandler context unless the downstream is within that context too\. Found task print-op-4 which depends on upstream task print-op-2 within an uncommon dsl\.ExitHandler context\.' ): - @dsl.component - def print_op(message: str): - print(message) - @dsl.pipeline(name='pipeline-with-multiple-exit-handlers') def my_pipeline(): first_exit_task = print_op(message='First exit task.') @@ -1965,10 +1948,6 @@ def test_exit_handler_task_prevented(self): r'Illegal task dependency across DSL context managers\. A downstream task cannot depend on an upstream task within a dsl\.ExitHandler context unless the downstream is within that context too\. Found task print-op-4 which depends on upstream task print-op-2 within an uncommon dsl\.ExitHandler context\.' ): - @dsl.component - def print_op(message: str): - print(message) - @dsl.pipeline(name='pipeline-with-multiple-exit-handlers') def my_pipeline(): first_exit_task = print_op(message='First exit task.') @@ -1989,10 +1968,6 @@ def my_pipeline(): def test_within_same_exit_handler_permitted(self): - @dsl.component - def print_op(message: str): - print(message) - @dsl.pipeline(name='pipeline-with-multiple-exit-handlers') def my_pipeline(): first_exit_task = print_op(message='First exit task.') @@ -2019,14 +1994,6 @@ def test_outside_of_condition_blocked(self): r'Illegal task dependency across DSL context managers\. A downstream task cannot depend on an upstream task within a dsl\.Condition context unless the downstream is within that context too\. Found task print-op-3 which depends on upstream task print-op within an uncommon dsl\.Condition context\.' ): - @dsl.component - def print_op(message: str): - print(message) - - @dsl.component - def return_1() -> int: - return 1 - @dsl.pipeline(name='pipeline-with-multiple-exit-handlers') def my_pipeline(): return_1_task = return_1() @@ -2043,14 +2010,6 @@ def my_pipeline(): def test_inside_of_condition_permitted(self): - @dsl.component - def print_op(message: str): - print(message) - - @dsl.component - def return_1() -> int: - return 1 - @dsl.pipeline(name='pipeline-with-multiple-exit-handlers') def my_pipeline(): return_1_task = return_1() @@ -2066,11 +2025,6 @@ def my_pipeline(): pipeline_func=my_pipeline, package_path=package_path) -@dsl.component -def identity(string: str, model: bool) -> str: - return string - - class TestYamlComments(unittest.TestCase): def test_comments_include_inputs_and_outputs_and_pipeline_name(self): @@ -2078,7 +2032,8 @@ def test_comments_include_inputs_and_outputs_and_pipeline_name(self): @dsl.pipeline() def my_pipeline(sample_input1: bool = True, sample_input2: str = 'string') -> str: - op1 = identity(string=sample_input2, model=sample_input1) + + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2113,7 +2068,7 @@ def test_no_description(self): @dsl.pipeline() def pipeline_with_no_description(sample_input1: bool = True, sample_input2: str = 'string') -> str: - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2152,7 +2107,7 @@ def test_description_from_docstring(self): def pipeline_with_description(sample_input1: bool = True, sample_input2: str = 'string') -> str: """This is a description of this pipeline.""" - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2191,7 +2146,7 @@ def test_description_from_decorator(self): def pipeline_with_description(sample_input1: bool = True, sample_input2: str = 'string') -> str: """Don't prefer this description.""" - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2229,7 +2184,7 @@ def test_comments_on_pipeline_with_no_inputs_or_outputs(self): @dsl.pipeline() def pipeline_with_no_inputs() -> str: - op1 = identity(string='string', model=True) + op1 = my_comp(string='string', model=True) result = op1.output return result @@ -2249,7 +2204,7 @@ def pipeline_with_no_inputs() -> str: @dsl.pipeline() def pipeline_with_no_outputs(sample_input1: bool = True, sample_input2: str = 'string'): - identity(string=sample_input2, model=sample_input1) + my_comp(string=sample_input2, model=sample_input1) with tempfile.TemporaryDirectory() as tmpdir: pipeline_spec_path = os.path.join(tmpdir, 'output.yaml') @@ -2270,7 +2225,7 @@ def test_comments_follow_pattern(self): def my_pipeline(sample_input1: bool = True, sample_input2: str = 'string') -> str: """This is a definition of this pipeline.""" - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2402,7 +2357,7 @@ def test_comments_idempotency(self): def my_pipeline(sample_input1: bool = True, sample_input2: str = 'string') -> str: """My description.""" - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2444,7 +2399,7 @@ def pipeline_with_multiline_definition( """docstring short description. docstring long description. docstring long description. """ - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2473,7 +2428,7 @@ def pipeline_with_multiline_definition( docstring long description. docstring long description. """ - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2503,7 +2458,7 @@ def my_pipeline(sample_input1: bool = True, docstring long description. docstring long description. """ - op1 = identity(string=sample_input2, model=sample_input1) + op1 = my_comp(string=sample_input2, model=sample_input1) result = op1.output return result @@ -2784,13 +2739,9 @@ def comp(x: Optional[Input[Artifact]] = None): def test_pipeline(self): - @dsl.component - def comp(): - print('hello') - @dsl.pipeline def my_pipeline(x: Optional[Input[Artifact]] = None): - comp() + print_hello() artifact_spec_from_root = my_pipeline.pipeline_spec.root.input_definitions.artifacts[ 'x'] @@ -2798,13 +2749,9 @@ def my_pipeline(x: Optional[Input[Artifact]] = None): def test_pipeline_without_optional_type_modifier(self): - @dsl.component - def comp(): - print('hello') - @dsl.pipeline def my_pipeline(x: Input[Artifact] = None): - comp() + print_hello() artifact_spec_from_root = my_pipeline.pipeline_spec.root.input_definitions.artifacts[ 'x'] @@ -2872,19 +2819,26 @@ def my_pipeline(x: Input[Artifact] = Artifact( class TestCrossTasksGroupFanInCollection(unittest.TestCase): - def test_missing_collected_with_correct_annotation(self): - from typing import List - - from kfp import dsl - - @dsl.component - def double(num: int) -> int: - return 2 * num + def test_correct_subdag_return_type(self): @dsl.component def add(nums: List[int]) -> int: return sum(nums) + @dsl.pipeline + def math_pipeline() -> int: + with dsl.ParallelFor([1, 2, 3]) as v: + t = double(num=v) + + return add(nums=dsl.Collected(t.output)).output + + self.assertEqual( + math_pipeline.pipeline_spec.components['comp-for-loop-2'] + .output_definitions.parameters['pipelinechannel--double-Output'] + .parameter_type, type_utils.LIST) + + def test_missing_collected_with_correct_annotation(self): + with self.assertRaisesRegex( type_utils.InconsistentTypeException, "Argument type 'NUMBER_INTEGER' is incompatible with the input type 'LIST'" @@ -2899,10 +2853,6 @@ def math_pipeline() -> int: def test_missing_collected_with_incorrect_annotation(self): - @dsl.component - def double(num: int) -> int: - return 2 * num - @dsl.component def add(nums: int) -> int: return nums @@ -2920,35 +2870,8 @@ def math_pipeline() -> int: return add(nums=t.output).output - def test_producer_condition_legal1(self): - from kfp import dsl - - @dsl.component - def double(num: int) -> int: - return 2 * num - - @dsl.component - def add(nums: List[int]) -> int: - return sum(nums) - - @dsl.pipeline - def math_pipeline(text: str) -> int: - with dsl.Condition(text == 'text'): - with dsl.ParallelFor([1, 2, 3]) as v: - t = double(num=v) - - return add(nums=dsl.Collected(t.output)).output - def test_producer_condition_legal2(self): - @dsl.component - def double(num: int) -> int: - return 2 * num - - @dsl.component - def add(nums: List[int]) -> int: - return sum(nums) - @dsl.pipeline def my_pipeline(a: str): with dsl.ParallelFor([1, 2, 3]) as v: @@ -2960,14 +2883,6 @@ def my_pipeline(a: str): def test_producer_condition_illegal1(self): - @dsl.component - def double(num: int) -> int: - return 2 * num - - @dsl.component - def add(nums: List[int]) -> int: - return sum(nums) - with self.assertRaisesRegex( compiler_utils.InvalidTopologyException, r'Illegal task dependency across DSL context managers\. When using dsl\.Collected to fan-in outputs from a task within a dsl\.ParallelFor context, the dsl\.ParallelFor context manager cannot be nested within a dsl.Condition context manager unless the consumer task is too\. Task add consumes from double within a dsl\.Condition context\.' @@ -2984,14 +2899,6 @@ def my_pipeline(a: str = '', b: str = ''): def test_producer_condition_illegal2(self): - @dsl.component - def double(num: int) -> int: - return 2 * num - - @dsl.component - def add(nums: List[int]) -> int: - return sum(nums) - with self.assertRaisesRegex( compiler_utils.InvalidTopologyException, r'Illegal task dependency across DSL context managers\. When using dsl\.Collected to fan-in outputs from a task within a dsl\.ParallelFor context, the dsl\.ParallelFor context manager cannot be nested within a dsl\.Condition context manager unless the consumer task is too\. Task add consumes from double within a dsl\.Condition context\.' @@ -3006,18 +2913,10 @@ def my_pipeline(a: str = ''): def test_producer_exit_handler_illegal1(self): - @dsl.component - def double(num: int) -> int: - return 2 * num - @dsl.component def exit_comp(): print('Running exit task!') - @dsl.component - def add(nums: List[int]) -> int: - return sum(nums) - with self.assertRaisesRegex( compiler_utils.InvalidTopologyException, r'Illegal task dependency across DSL context managers\. When using dsl\.Collected to fan-in outputs from a task within a dsl\.ParallelFor context, the dsl\.ParallelFor context manager cannot be nested within a dsl\.ExitHandler context manager unless the consumer task is too\. Task add consumes from double within a dsl\.ExitHandler context\.' @@ -3070,18 +2969,10 @@ def my_pipeline(): def test_producer_and_consumer_in_same_context(self): - @dsl.component - def double(num: int) -> int: - return 2 * num - - @dsl.component - def add(nums: List[int]) -> int: - return sum(nums) - - with self.assertRaisesRegex( - compiler_utils.InvalidTopologyException, - r'dsl\.Collected can only be used to fan-in outputs produced by a task within a dsl\.ParallelFor context to a task outside of the dsl\.ParallelFor context\. Producer task double is either not in a dsl\.ParallelFor context or is only in a dsl\.ParallelFor that also contains consumer task add\.' - ): + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r'dsl\.Collected can only be used to fan-in outputs produced by a task within a dsl\.ParallelFor context to a task outside of the dsl\.ParallelFor context\. Producer task double is either not in a dsl\.ParallelFor context or is only in a dsl\.ParallelFor that also contains consumer task add\.' + ): @dsl.pipeline def math_pipeline(): @@ -3091,14 +2982,6 @@ def math_pipeline(): def test_no_parallelfor_context(self): - @dsl.component - def double(num: int) -> int: - return 2 * num - - @dsl.component - def add(nums: List[int]) -> int: - return sum(nums) - with self.assertRaisesRegex( compiler_utils.InvalidTopologyException, r'dsl\.Collected can only be used to fan-in outputs produced by a task within a dsl\.ParallelFor context to a task outside of the dsl\.ParallelFor context\. Producer task double is either not in a dsl\.ParallelFor context or is only in a dsl\.ParallelFor that also contains consumer task add\.' @@ -3138,6 +3021,141 @@ def my_pipeline(sample_input1: str = 'message'): my_pipeline.pipeline_spec.root.dag.tasks['fail-op'].trigger_policy .strategy, 0) + def test_can_use_task_final_status(self): + + @dsl.component + def worker_component() -> str: + return 'hello' + + @dsl.component + def cancel_handler( + status: PipelineTaskFinalStatus, + text: str = '', + ): + print(text) + print(status) + + @dsl.pipeline + def my_pipeline(): + worker_task = worker_component() + exit_task = cancel_handler( + text=worker_task.output).ignore_upstream_failure() + + self.assertEqual( + my_pipeline.pipeline_spec.root.dag.tasks['cancel-handler'] + .trigger_policy.strategy, 2) + self.assertEqual( + my_pipeline.pipeline_spec.root.dag.tasks['cancel-handler'].inputs + .parameters['status'].task_final_status.producer_task, + 'worker-component') + + status_param = my_pipeline.pipeline_spec.components[ + 'comp-cancel-handler'].input_definitions.parameters['status'] + self.assertTrue(status_param.is_optional) + self.assertEqual(status_param.parameter_type, + type_utils.TASK_FINAL_STATUS) + + self.assertEqual( + my_pipeline.pipeline_spec.root.dag.tasks['worker-component'] + .trigger_policy.strategy, 0) + + def test_cannot_use_task_final_status_under_task_group(self): + + @dsl.component + def worker_component() -> str: + return 'hello' + + @dsl.component + def cancel_handler( + status: PipelineTaskFinalStatus, + text: str = '', + ): + print(text) + print(status) + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r"Tasks that use '\.ignore_upstream_failure\(\)' and 'PipelineTaskFinalStatus' must have exactly one dependent upstream task within the same control flow scope\. Got task 'cancel-handler' beneath a 'dsl\.Condition' that does not also contain the upstream dependent task\.", + ): + + @dsl.pipeline + def my_pipeline(): + worker_task = worker_component() + with dsl.Condition(worker_task.output == 'foo'): + exit_task = cancel_handler( + text=worker_task.output).ignore_upstream_failure() + + def test_cannot_use_final_task_status_if_zero_dependencies(self): + + @dsl.component + def worker_component() -> str: + return 'hello' + + @dsl.component + def cancel_handler(status: PipelineTaskFinalStatus,): + print(status) + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r"Tasks that use '\.ignore_upstream_failure\(\)' and 'PipelineTaskFinalStatus' must have exactly one dependent upstream task\. Got task 'cancel-handler with no upstream dependencies\.", + ): + + @dsl.pipeline + def my_pipeline(): + worker_task = worker_component() + exit_task = cancel_handler().ignore_upstream_failure() + + def test_cannot_use_task_final_status_if_more_than_one_dependency_implicit( + self): + + @dsl.component + def worker_component() -> str: + return 'hello' + + @dsl.component + def cancel_handler( + status: PipelineTaskFinalStatus, + a: str = '', + b: str = '', + ): + print(status) + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r"Tasks that use '\.ignore_upstream_failure\(\)' and 'PipelineTaskFinalStatus' must have exactly one dependent upstream task\. Got 2 dependent tasks: \['worker-component', 'worker-component-2']\.", + ): + + @dsl.pipeline + def my_pipeline(): + worker_task1 = worker_component() + worker_task2 = worker_component() + exit_task = cancel_handler( + a=worker_task1.output, + b=worker_task2.output).ignore_upstream_failure() + + def test_cannot_use_task_final_status_if_more_than_one_dependency_explicit( + self): + + @dsl.component + def worker_component() -> str: + return 'hello' + + @dsl.component + def cancel_handler(status: PipelineTaskFinalStatus,): + print(status) + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r"Tasks that use '\.ignore_upstream_failure\(\)' and 'PipelineTaskFinalStatus' must have exactly one dependent upstream task\. Got 2 dependent tasks: \['worker-component', 'worker-component-2']\.", + ): + + @dsl.pipeline + def my_pipeline(): + worker_task1 = worker_component() + worker_task2 = worker_component() + exit_task = cancel_handler().after( + worker_task1, worker_task2).ignore_upstream_failure() + def test_component_with_no_input_permitted(self): @dsl.component @@ -3203,13 +3221,9 @@ def fail_op(message: str = 'message') -> str: sys.exit(1) return message - @dsl.component - def identity(message: str = 'message') -> str: - return message - @dsl.pipeline def wrapped_pipeline(message: str = 'message') -> str: - task = identity(message=message) + task = print_and_return(text=message) return task.output @dsl.pipeline @@ -3235,10 +3249,6 @@ def fail_op(message: str) -> str: sys.exit(1) return message - @dsl.component - def print_op(message: str): - print(message) - with self.assertRaisesRegex( ValueError, r'Tasks can only use .ignore_upstream_failure()'): @@ -3258,10 +3268,6 @@ def fail_op(message: str) -> str: sys.exit(1) return message - @dsl.component - def print_op(message: str): - print(message) - @dsl.pipeline() def my_pipeline(sample_input1: str = 'message'): task = fail_op(message=sample_input1) @@ -3284,10 +3290,6 @@ def fail_op(message: str) -> str: sys.exit(1) return message - @dsl.component - def print_op(message: str): - print(message) - @dsl.pipeline() def my_pipeline(sample_input1: str = 'message'): task = fail_op(message=sample_input1) @@ -3436,11 +3438,6 @@ def baz_platform_set_bat_feature(task: pipeline_task.PipelineTask, return task -@dsl.component -def comp(): - pass - - def compile_and_reload( pipeline: graph_component.GraphComponent ) -> yaml_component.YamlComponent: @@ -3455,49 +3452,45 @@ class TestResourceConfig(unittest.TestCase): def test_cpu_memory_optional(self): - @dsl.component - def predict_op() -> str: - return 'a' - @dsl.pipeline def simple_pipeline(): - predict_op() - predict_op().set_cpu_limit('5') - predict_op().set_memory_limit('50G') - predict_op().set_cpu_request('2').set_cpu_limit( + return_1() + return_1().set_cpu_limit('5') + return_1().set_memory_limit('50G') + return_1().set_cpu_request('2').set_cpu_limit( '5').set_memory_request('4G').set_memory_limit('50G') dict_format = json_format.MessageToDict(simple_pipeline.pipeline_spec) self.assertNotIn( 'resources', dict_format['deploymentSpec']['executors'] - ['exec-predict-op']['container']) + ['exec-return-1']['container']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-predict-op-2'] + 5, dict_format['deploymentSpec']['executors']['exec-return-1-2'] ['container']['resources']['cpuLimit']) self.assertNotIn( 'memoryLimit', dict_format['deploymentSpec']['executors'] - ['exec-predict-op-2']['container']['resources']) + ['exec-return-1-2']['container']['resources']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-predict-op-3'] + 50, dict_format['deploymentSpec']['executors']['exec-return-1-3'] ['container']['resources']['memoryLimit']) self.assertNotIn( 'cpuLimit', dict_format['deploymentSpec']['executors'] - ['exec-predict-op-3']['container']['resources']) + ['exec-return-1-3']['container']['resources']) self.assertEqual( - 2, dict_format['deploymentSpec']['executors']['exec-predict-op-4'] + 2, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['cpuRequest']) self.assertEqual( - 5, dict_format['deploymentSpec']['executors']['exec-predict-op-4'] + 5, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['cpuLimit']) self.assertEqual( - 4, dict_format['deploymentSpec']['executors']['exec-predict-op-4'] + 4, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['memoryRequest']) self.assertEqual( - 50, dict_format['deploymentSpec']['executors']['exec-predict-op-4'] + 50, dict_format['deploymentSpec']['executors']['exec-return-1-4'] ['container']['resources']['memoryLimit']) @@ -4173,5 +4166,1558 @@ def my_pipeline( 'Component output artifact.') +class TestCannotReturnFromWithinControlFlowGroup(unittest.TestCase): + + def test_condition_raises(self): + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r'Pipeline outputs may only be returned from the top level of the pipeline function scope\. Got pipeline output from within the control flow group dsl\.Condition\.' + ): + + @dsl.pipeline + def my_pipeline(string: str = 'string') -> str: + with dsl.Condition(string == 'foo'): + return print_and_return(text=string).output + + def test_loop_raises(self): + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r'Pipeline outputs may only be returned from the top level of the pipeline function scope\. Got pipeline output from within the control flow group dsl\.ParallelFor\.' + ): + + @dsl.pipeline + def my_pipeline(string: str = 'string') -> str: + with dsl.ParallelFor([1, 2, 3]): + return print_and_return(text=string).output + + def test_exit_handler_raises(self): + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + r'Pipeline outputs may only be returned from the top level of the pipeline function scope\. Got pipeline output from within the control flow group dsl\.ExitHandler\.' + ): + + @dsl.pipeline + def my_pipeline(string: str = 'string') -> str: + with dsl.ExitHandler(print_and_return(text='exit task')): + return print_and_return(text=string).output + + +class TestConditionLogic(unittest.TestCase): + + def test_if(self): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads!') + + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag.tasks['condition-1'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads'" + ) + + def test_if_else(self): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads!') + with dsl.Else(): + print_and_return(text='Got tails!') + + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-2'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads'" + ) + + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-3'] + .trigger_policy.condition, + "!(inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads')" + ) + + def test_if_elif_else(self): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + print_and_return(text='Got tails!') + with dsl.Else(): + print_and_return(text='Draw!') + + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-2'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--roll-three-sided-die-Output'] == 'heads'" + ) + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-3'] + .trigger_policy.condition, + "!(inputs.parameter_values['pipelinechannel--roll-three-sided-die-Output'] == 'heads') && inputs.parameter_values['pipelinechannel--roll-three-sided-die-Output'] == 'tails'" + ) + + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-4'] + .trigger_policy.condition, + "!(inputs.parameter_values['pipelinechannel--roll-three-sided-die-Output'] == 'heads') && !(inputs.parameter_values['pipelinechannel--roll-three-sided-die-Output'] == 'tails')" + ) + + def test_if_multiple_elif_else(self): + + @dsl.pipeline + def int_to_string(): + int_task = int_zero_through_three() + with dsl.If(int_task.output == 0): + print_and_return(text='Got zero!') + with dsl.Elif(int_task.output == 1): + print_and_return(text='Got one!') + with dsl.Elif(int_task.output == 2): + print_and_return(text='Got two!') + with dsl.Else(): + print_and_return(text='Got three!') + + self.assertEqual( + int_to_string.pipeline_spec.components['comp-condition-branches-1'] + .dag.tasks['condition-2'].trigger_policy.condition, + "int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 0" + ) + self.assertEqual( + int_to_string.pipeline_spec.components['comp-condition-branches-1'] + .dag.tasks['condition-3'].trigger_policy.condition, + "!(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 0) && int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 1" + ) + self.assertEqual( + int_to_string.pipeline_spec.components['comp-condition-branches-1'] + .dag.tasks['condition-4'].trigger_policy.condition, + "!(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 0) && !(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 1) && int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 2" + ) + self.assertEqual( + int_to_string.pipeline_spec.components['comp-condition-branches-1'] + .dag.tasks['condition-5'].trigger_policy.condition, + "!(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 0) && !(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 1) && !(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 2)" + ) + + def test_nested_if_elif_else_with_pipeline_param(self): + + @dsl.pipeline + def flip_coin_pipeline(confirm: bool): + int_task = int_zero_through_three() + heads_task = flip_coin() + + with dsl.If(heads_task.output == 'heads'): + with dsl.If(int_task.output == 0): + print_and_return(text='Got zero!') + + with dsl.Elif(int_task.output == 1): + task = print_and_return(text='Got one!') + with dsl.If(confirm == True): + print_and_return(text='Confirmed: definitely got one.') + + with dsl.Elif(int_task.output == 2): + print_and_return(text='Got two!') + + with dsl.Else(): + print_and_return(text='Got three!') + + # tests that the pipeline wrapper works well with multiple if/elif/else + with dsl.ParallelFor(['Game #1', 'Game #2']) as game_no: + heads_task = flip_coin() + with dsl.If(heads_task.output == 'heads'): + print_and_return(text=game_no) + print_and_return(text='Got heads!') + with dsl.Else(): + print_and_return(text=game_no) + print_and_return(text='Got tail!') + + # first group + ## top level conditions + ### if + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag.tasks['condition-1'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads'" + ) + ## second level nested conditions + ### if + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-2'].dag.tasks['condition-3'] + .trigger_policy.condition, + "int(inputs.parameter_values[\'pipelinechannel--int-zero-through-three-Output\']) == 0" + ) + ### elif + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-2'].dag.tasks['condition-4'] + .trigger_policy.condition, + "!(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 0) && int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 1" + ) + ### elif #2 + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-2'].dag.tasks['condition-6'] + .trigger_policy.condition, + "!(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 0) && !(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 1) && int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 2" + ) + ### else + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-2'].dag.tasks['condition-7'] + .trigger_policy.condition, + "!(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 0) && !(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 1) && !(int(inputs.parameter_values['pipelinechannel--int-zero-through-three-Output']) == 2)" + ) + ## third level nested conditions + ### if + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-4'].dag + .tasks['condition-5'].trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--confirm'] == true") + + # second group + + ## if + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-10'].dag.tasks['condition-11'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--flip-coin-2-Output'] == 'heads'" + ) + ## elif + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-10'].dag.tasks['condition-12'] + .trigger_policy.condition, + "!(inputs.parameter_values['pipelinechannel--flip-coin-2-Output'] == 'heads')" + ) + + def test_multiple_ifs_permitted(self): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads!') + with dsl.If(flip_coin_task.output == 'tails'): + print_and_return(text='Got tails!') + + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag.tasks['condition-1'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads'" + ) + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag.tasks['condition-2'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'tails'" + ) + + def test_multiple_else_not_permitted(self): + with self.assertRaisesRegex( + tasks_group.InvalidControlFlowException, + r'Cannot use dsl\.Else following another dsl\.Else\. dsl\.Else can only be used following an upstream dsl\.If or dsl\.Elif\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads!') + with dsl.Else(): + print_and_return(text='Got tails!') + with dsl.Else(): + print_and_return(text='Got tails!') + + def test_else_no_if_not_supported(self): + with self.assertRaisesRegex( + tasks_group.InvalidControlFlowException, + r'dsl\.Else can only be used following an upstream dsl\.If or dsl\.Elif\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(): + with dsl.Else(): + print_and_return(text='Got unknown') + + def test_elif_no_if_not_supported(self): + with self.assertRaisesRegex( + tasks_group.InvalidControlFlowException, + r'dsl\.Elif can only be used following an upstream dsl\.If or dsl\.Elif\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = flip_coin() + with dsl.Elif(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads!') + + def test_boolean_condition_has_helpful_error(self): + with self.assertRaisesRegex( + ValueError, + r'Got constant boolean True as a condition\. This is likely because the provided condition evaluated immediately\. At least one of the operands must be an output from an upstream task or a pipeline parameter\.' + ): + + @dsl.pipeline + def my_pipeline(): + with dsl.Condition('foo' == 'foo'): + print_and_return(text='I will always run.') + + def test_boolean_elif_has_helpful_error(self): + with self.assertRaisesRegex( + ValueError, + r'Got constant boolean False as a condition\. This is likely because the provided condition evaluated immediately\. At least one of the operands must be an output from an upstream task or a pipeline parameter\.' + ): + + @dsl.pipeline + def my_pipeline(text: str): + with dsl.If(text == 'foo'): + print_and_return(text='I will always run.') + with dsl.Elif('foo' == 'bar'): + print_and_return(text='I will never run.') + + def test_tasks_instantiated_between_if_else_and_elif_permitted(self): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads on coin one!') + + flip_coin_task_2 = flip_coin() + + with dsl.Elif(flip_coin_task_2.output == 'tails'): + print_and_return(text='Got heads on coin two!') + + flip_coin_task_3 = flip_coin() + + with dsl.Else(): + print_and_return( + text=f'Coin three result: {flip_coin_task_3.output}') + + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-2'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads'" + ) + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-3'] + .trigger_policy.condition, + "!(inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads') && inputs.parameter_values['pipelinechannel--flip-coin-2-Output'] == 'tails'" + ) + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].dag.tasks['condition-4'] + .trigger_policy.condition, + "!(inputs.parameter_values['pipelinechannel--flip-coin-Output'] == 'heads') && !(inputs.parameter_values['pipelinechannel--flip-coin-2-Output'] == 'tails')" + ) + + def test_other_control_flow_instantiated_between_if_else_not_permitted( + self): + with self.assertRaisesRegex( + tasks_group.InvalidControlFlowException, + 'dsl\.Else can only be used following an upstream dsl\.If or dsl\.Elif\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_and_return(text='Got heads!') + with dsl.ParallelFor(['foo', 'bar']) as item: + print_and_return(text=item) + with dsl.Else(): + print_and_return(text='Got tails!') + + +class TestDslOneOf(unittest.TestCase): + # The space of possible tests is very large, so we test a representative set of cases covering the following styles of usage: + # - upstream conditions: if/else v if/elif/else + # - data consumed: parameters v artifacts + # - where dsl.OneOf goes: consumed by task v returned v both + # - when outputs have different keys: e.g., .output v .outputs[] + # - how the if/elif/else are nested and at what level they are consumed + + # Data type validation (e.g., dsl.OneOf(artifact, param) fails) and similar is covered in pipeline_channel_test.py. + + # To help narrow the tests further (we already test lots of aspects in the following cases), we choose focus on the dsl.OneOf behavior, not the conditional logic if If/Elif/Else. This is more verbose, but more maintainable and the behavior under test is clearer. + + def test_if_else_returned(self): + """Uses If and Else branches, parameters passed to dsl.OneOf, dsl.OneOf returned from a pipeline, and different output keys on dsl.OneOf channels.""" + + @dsl.pipeline + def roll_die_pipeline() -> str: + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Else(): + t2 = print_and_return_with_output_key(text='Got tails!') + return dsl.OneOf(t1.output, t2.outputs['output_key']) + + # hole punched through if + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-2'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-Output'].parameter_type, + type_utils.STRING, + ) + # hole punched through else + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-3'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-with-output-key-output_key'] + .parameter_type, + type_utils.STRING, + ) + # condition-branches surfaces + self.assertEqual( + roll_die_pipeline.pipeline_spec + .components['comp-condition-branches-1'].output_definitions + .parameters['pipelinechannel--condition-branches-1-oneof-1'] + .parameter_type, + type_utils.STRING, + ) + parameter_selectors = roll_die_pipeline.pipeline_spec.components[ + 'comp-condition-branches-1'].dag.outputs.parameters[ + 'pipelinechannel--condition-branches-1-oneof-1'].value_from_oneof.parameter_selectors + + self.assertEqual( + parameter_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-Output', + producer_subtask='condition-2', + )) + self.assertEqual( + parameter_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-with-output-key-output_key', + producer_subtask='condition-3', + )) + # surfaced as output + self.assertEqual( + roll_die_pipeline.pipeline_spec.root.dag.outputs + .parameters['Output'].value_from_parameter, + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + producer_subtask='condition-branches-1', + output_parameter_key='pipelinechannel--condition-branches-1-oneof-1', + ), + ) + + def test_if_elif_else_returned(self): + """Uses If, Elif, and Else branches, parameters passed to dsl.OneOf, dsl.OneOf returned from a pipeline, and different output keys on dsl.OneOf channels.""" + + @dsl.pipeline + def roll_die_pipeline() -> str: + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + with dsl.Else(): + t3 = print_and_return_with_output_key(text='Draw!') + return dsl.OneOf(t1.output, t2.output, t3.outputs['output_key']) + + # hole punched through if + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-2'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-Output'].parameter_type, + type_utils.STRING, + ) + # hole punched through elif + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-3'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-2-Output'].parameter_type, + type_utils.STRING, + ) + # hole punched through else + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-4'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-with-output-key-output_key'] + .parameter_type, + type_utils.STRING, + ) + # condition-branches surfaces + self.assertEqual( + roll_die_pipeline.pipeline_spec + .components['comp-condition-branches-1'].output_definitions + .parameters['pipelinechannel--condition-branches-1-oneof-1'] + .parameter_type, + type_utils.STRING, + ) + parameter_selectors = roll_die_pipeline.pipeline_spec.components[ + 'comp-condition-branches-1'].dag.outputs.parameters[ + 'pipelinechannel--condition-branches-1-oneof-1'].value_from_oneof.parameter_selectors + self.assertEqual( + parameter_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-Output', + producer_subtask='condition-2', + )) + self.assertEqual( + parameter_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-2-Output', + producer_subtask='condition-3', + )) + self.assertEqual( + parameter_selectors[2], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-with-output-key-output_key', + producer_subtask='condition-4', + )) + # surfaced as output + self.assertEqual( + roll_die_pipeline.pipeline_spec.root.dag.outputs + .parameters['Output'].value_from_parameter, + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + producer_subtask='condition-branches-1', + output_parameter_key='pipelinechannel--condition-branches-1-oneof-1', + ), + ) + + def test_if_elif_else_consumed(self): + """Uses If, Elif, and Else branches, parameters passed to dsl.OneOf, dsl.OneOf passed to a consumer task, and different output keys on dsl.OneOf channels.""" + + @dsl.pipeline + def roll_die_pipeline(): + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + with dsl.Else(): + t3 = print_and_return_with_output_key(text='Draw!') + print_and_return( + text=dsl.OneOf(t1.output, t2.output, t3.outputs['output_key'])) + + # hole punched through if + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-2'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-Output'].parameter_type, + type_utils.STRING, + ) + # hole punched through elif + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-3'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-2-Output'].parameter_type, + type_utils.STRING, + ) + # hole punched through else + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-4'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-with-output-key-output_key'] + .parameter_type, + type_utils.STRING, + ) + # condition-branches surfaces + self.assertEqual( + roll_die_pipeline.pipeline_spec + .components['comp-condition-branches-1'].output_definitions + .parameters['pipelinechannel--condition-branches-1-oneof-1'] + .parameter_type, + type_utils.STRING, + ) + parameter_selectors = roll_die_pipeline.pipeline_spec.components[ + 'comp-condition-branches-1'].dag.outputs.parameters[ + 'pipelinechannel--condition-branches-1-oneof-1'].value_from_oneof.parameter_selectors + self.assertEqual( + parameter_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-Output', + producer_subtask='condition-2', + )) + self.assertEqual( + parameter_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-2-Output', + producer_subtask='condition-3', + )) + self.assertEqual( + parameter_selectors[2], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-with-output-key-output_key', + producer_subtask='condition-4', + )) + # consumed from condition-branches + self.assertEqual( + roll_die_pipeline.pipeline_spec.root.dag.tasks['print-and-return-3'] + .inputs.parameters['text'].task_output_parameter, + pipeline_spec_pb2.TaskInputsSpec.InputParameterSpec + .TaskOutputParameterSpec( + producer_task='condition-branches-1', + output_parameter_key='pipelinechannel--condition-branches-1-oneof-1', + ), + ) + + def test_if_else_consumed_and_returned(self): + """Uses If, Elif, and Else branches, parameters passed to dsl.OneOf, and dsl.OneOf passed to a consumer task and returned from the pipeline.""" + + @dsl.pipeline + def flip_coin_pipeline() -> str: + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return(text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return(text='Got tails!') + x = dsl.OneOf(print_task_1.output, print_task_2.output) + print_and_return(text=x) + return x + + # hole punched through if + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-2'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-Output'].parameter_type, + type_utils.STRING, + ) + # hole punched through else + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-3'] + .output_definitions.parameters[ + 'pipelinechannel--print-and-return-2-Output'].parameter_type, + type_utils.STRING, + ) + # condition-branches surfaces + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].output_definitions + .parameters['pipelinechannel--condition-branches-1-oneof-1'] + .parameter_type, + type_utils.STRING, + ) + parameter_selectors = flip_coin_pipeline.pipeline_spec.components[ + 'comp-condition-branches-1'].dag.outputs.parameters[ + 'pipelinechannel--condition-branches-1-oneof-1'].value_from_oneof.parameter_selectors + self.assertEqual( + parameter_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-Output', + producer_subtask='condition-2', + )) + self.assertEqual( + parameter_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-2-Output', + producer_subtask='condition-3', + )) + # consumed from condition-branches + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag + .tasks['print-and-return-3'].inputs.parameters['text'] + .task_output_parameter, + pipeline_spec_pb2.TaskInputsSpec.InputParameterSpec + .TaskOutputParameterSpec( + producer_task='condition-branches-1', + output_parameter_key='pipelinechannel--condition-branches-1-oneof-1', + ), + ) + + # surfaced as output + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag.outputs + .parameters['Output'].value_from_parameter, + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + producer_subtask='condition-branches-1', + output_parameter_key='pipelinechannel--condition-branches-1-oneof-1', + ), + ) + + def test_if_else_consumed_and_returned_artifacts(self): + """Uses If, Elif, and Else branches, artifacts passed to dsl.OneOf, and dsl.OneOf passed to a consumer task and returned from the pipeline.""" + + @dsl.pipeline + def flip_coin_pipeline() -> Artifact: + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return_as_artifact(text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return_as_artifact(text='Got tails!') + x = dsl.OneOf(print_task_1.outputs['a'], print_task_2.outputs['a']) + print_artifact(a=x) + return x + + # hole punched through if + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-2'] + .output_definitions + .artifacts['pipelinechannel--print-and-return-as-artifact-a'] + .artifact_type.schema_title, + 'system.Artifact', + ) + # hole punched through else + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-3'] + .output_definitions + .artifacts['pipelinechannel--print-and-return-as-artifact-2-a'] + .artifact_type.schema_title, + 'system.Artifact', + ) + # condition-branches surfaces + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-1'].output_definitions + .artifacts['pipelinechannel--condition-branches-1-oneof-1'] + .artifact_type.schema_title, + 'system.Artifact', + ) + artifact_selectors = flip_coin_pipeline.pipeline_spec.components[ + 'comp-condition-branches-1'].dag.outputs.artifacts[ + 'pipelinechannel--condition-branches-1-oneof-1'].artifact_selectors + self.assertEqual( + artifact_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ArtifactSelectorSpec( + output_artifact_key='pipelinechannel--print-and-return-as-artifact-a', + producer_subtask='condition-2', + )) + self.assertEqual( + artifact_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ArtifactSelectorSpec( + output_artifact_key='pipelinechannel--print-and-return-as-artifact-2-a', + producer_subtask='condition-3', + )) + + # consumed from condition-branches + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag.tasks['print-artifact'] + .inputs.artifacts['a'].task_output_artifact, + pipeline_spec_pb2.TaskInputsSpec.InputArtifactSpec + .TaskOutputArtifactSpec( + producer_task='condition-branches-1', + output_artifact_key='pipelinechannel--condition-branches-1-oneof-1', + ), + ) + + # surfaced as output + self.assertEqual( + flip_coin_pipeline.pipeline_spec.root.dag.outputs + .artifacts['Output'].artifact_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ArtifactSelectorSpec( + producer_subtask='condition-branches-1', + output_artifact_key='pipelinechannel--condition-branches-1-oneof-1', + ), + ) + + def test_nested_under_condition_consumed(self): + """Uses If, Else, and OneOf nested under a parent If.""" + + @dsl.pipeline + def flip_coin_pipeline(execute_pipeline: bool): + with dsl.If(execute_pipeline == True): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return_as_artifact( + text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return_as_artifact( + text='Got tails!') + x = dsl.OneOf(print_task_1.outputs['a'], + print_task_2.outputs['a']) + print_artifact(a=x) + + # hole punched through if + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-3'] + .output_definitions + .artifacts['pipelinechannel--print-and-return-as-artifact-a'] + .artifact_type.schema_title, + 'system.Artifact', + ) + # hole punched through else + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-4'] + .output_definitions + .artifacts['pipelinechannel--print-and-return-as-artifact-2-a'] + .artifact_type.schema_title, + 'system.Artifact', + ) + # condition-branches surfaces + self.assertEqual( + flip_coin_pipeline.pipeline_spec + .components['comp-condition-branches-2'].output_definitions + .artifacts['pipelinechannel--condition-branches-2-oneof-1'] + .artifact_type.schema_title, + 'system.Artifact', + ) + artifact_selectors = flip_coin_pipeline.pipeline_spec.components[ + 'comp-condition-branches-2'].dag.outputs.artifacts[ + 'pipelinechannel--condition-branches-2-oneof-1'].artifact_selectors + self.assertEqual( + artifact_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ArtifactSelectorSpec( + output_artifact_key='pipelinechannel--print-and-return-as-artifact-a', + producer_subtask='condition-3', + )) + self.assertEqual( + artifact_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ArtifactSelectorSpec( + output_artifact_key='pipelinechannel--print-and-return-as-artifact-2-a', + producer_subtask='condition-4', + )) + # consumed from condition-branches + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-1'].dag + .tasks['print-artifact'].inputs.artifacts['a'].task_output_artifact, + pipeline_spec_pb2.TaskInputsSpec.InputArtifactSpec + .TaskOutputArtifactSpec( + producer_task='condition-branches-2', + output_artifact_key='pipelinechannel--condition-branches-2-oneof-1', + ), + ) + + def test_nested_under_condition_returned_raises(self): + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + f'Pipeline outputs may only be returned from the top level of the pipeline function scope\. Got pipeline output dsl\.OneOf from within the control flow group dsl\.If\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(execute_pipeline: bool): + with dsl.If(execute_pipeline == True): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return_as_artifact( + text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return_as_artifact( + text='Got tails!') + return dsl.OneOf(print_task_1.outputs['a'], + print_task_2.outputs['a']) + + def test_deeply_nested_consumed(self): + """Uses If, Elif, Else, and OneOf deeply nested within multiple dub-DAGs.""" + + @dsl.pipeline + def flip_coin_pipeline(execute_pipeline: bool): + with dsl.ExitHandler(cleanup()): + with dsl.ParallelFor([1, 2, 3]): + with dsl.If(execute_pipeline == True): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return_as_artifact( + text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return_as_artifact( + text='Got tails!') + x = dsl.OneOf(print_task_1.outputs['a'], + print_task_2.outputs['a']) + print_artifact(a=x) + + self.assertIn( + 'condition-branches-5', flip_coin_pipeline.pipeline_spec + .components['comp-condition-4'].dag.tasks) + # consumed from condition-branches + self.assertEqual( + flip_coin_pipeline.pipeline_spec.components['comp-condition-4'].dag + .tasks['print-artifact'].inputs.artifacts['a'].task_output_artifact, + pipeline_spec_pb2.TaskInputsSpec.InputArtifactSpec + .TaskOutputArtifactSpec( + producer_task='condition-branches-5', + output_artifact_key='pipelinechannel--condition-branches-5-oneof-1', + ), + ) + + def test_deeply_nested_returned_raises(self): + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + f'Pipeline outputs may only be returned from the top level of the pipeline function scope\. Got pipeline output dsl\.OneOf from within the control flow group dsl\.ParallelFor\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(execute_pipeline: bool) -> str: + with dsl.ExitHandler(cleanup()): + with dsl.If(execute_pipeline == True): + with dsl.ParallelFor([1, 2, 3]): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return_as_artifact( + text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return_as_artifact( + text='Got tails!') + return dsl.OneOf(print_task_1.outputs['a'], + print_task_2.outputs['a']) + + def test_consume_at_wrong_level(self): + + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + f'Illegal task dependency across DSL context managers\. A downstream task cannot depend on an upstream task within a dsl\.If context unless the downstream is within that context too\. Found task print-artifact which depends on upstream task condition-branches-5 within an uncommon dsl\.If context\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(execute_pipeline: bool): + with dsl.ExitHandler(cleanup()): + with dsl.ParallelFor([1, 2, 3]): + with dsl.If(execute_pipeline == True): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return_as_artifact( + text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return_as_artifact( + text='Got tails!') + x = dsl.OneOf(print_task_1.outputs['a'], + print_task_2.outputs['a']) + # this is one level dedented from the permitted case + print_artifact(a=x) + + def test_return_at_wrong_level(self): + with self.assertRaisesRegex( + compiler_utils.InvalidTopologyException, + f'Pipeline outputs may only be returned from the top level of the pipeline function scope\. Got pipeline output dsl\.OneOf from within the control flow group dsl\.If\.' + ): + + @dsl.pipeline + def flip_coin_pipeline(execute_pipeline: bool): + with dsl.If(execute_pipeline == True): + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return_as_artifact( + text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return_as_artifact( + text='Got tails!') + # this is returned at the right level, but not permitted since it's still effectively returning from within the dsl.If group + return dsl.OneOf(print_task_1.outputs['a'], + print_task_2.outputs['a']) + + def test_oneof_in_condition(self): + """Tests that dsl.OneOf's channel can be consumed in a downstream group nested one level""" + + @dsl.pipeline + def roll_die_pipeline(repeat_on: str = 'Got heads!'): + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + with dsl.Else(): + t3 = print_and_return_with_output_key(text='Draw!') + x = dsl.OneOf(t1.output, t2.output, t3.outputs['output_key']) + + with dsl.If(x == repeat_on): + print_and_return(text=x) + + # condition-branches surfaces + self.assertEqual( + roll_die_pipeline.pipeline_spec + .components['comp-condition-branches-1'].output_definitions + .parameters['pipelinechannel--condition-branches-1-oneof-1'] + .parameter_type, + type_utils.STRING, + ) + parameter_selectors = roll_die_pipeline.pipeline_spec.components[ + 'comp-condition-branches-1'].dag.outputs.parameters[ + 'pipelinechannel--condition-branches-1-oneof-1'].value_from_oneof.parameter_selectors + self.assertEqual( + parameter_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-Output', + producer_subtask='condition-2', + )) + self.assertEqual( + parameter_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-2-Output', + producer_subtask='condition-3', + )) + self.assertEqual( + parameter_selectors[2], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-with-output-key-output_key', + producer_subtask='condition-4', + )) + # condition points to correct upstream output + self.assertEqual( + roll_die_pipeline.pipeline_spec.root.dag.tasks['condition-5'] + .trigger_policy.condition, + "inputs.parameter_values['pipelinechannel--condition-branches-1-pipelinechannel--condition-branches-1-oneof-1'] == inputs.parameter_values['pipelinechannel--repeat_on']" + ) + + def test_consumed_in_nested_groups(self): + """Tests that dsl.OneOf's channel can be consumed in a downstream group nested multiple levels""" + + @dsl.pipeline + def roll_die_pipeline( + repeat: bool = True, + rounds: List[str] = ['a', 'b', 'c'], + ): + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + with dsl.Else(): + t3 = print_and_return_with_output_key(text='Draw!') + x = dsl.OneOf(t1.output, t2.output, t3.outputs['output_key']) + + with dsl.ParallelFor(rounds): + with dsl.If(repeat == True): + print_and_return(text=x) + + # condition-branches surfaces + self.assertEqual( + roll_die_pipeline.pipeline_spec + .components['comp-condition-branches-1'].output_definitions + .parameters['pipelinechannel--condition-branches-1-oneof-1'] + .parameter_type, + type_utils.STRING, + ) + parameter_selectors = roll_die_pipeline.pipeline_spec.components[ + 'comp-condition-branches-1'].dag.outputs.parameters[ + 'pipelinechannel--condition-branches-1-oneof-1'].value_from_oneof.parameter_selectors + self.assertEqual( + parameter_selectors[0], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-Output', + producer_subtask='condition-2', + )) + self.assertEqual( + parameter_selectors[1], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-2-Output', + producer_subtask='condition-3', + )) + self.assertEqual( + parameter_selectors[2], + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + output_parameter_key='pipelinechannel--print-and-return-with-output-key-output_key', + producer_subtask='condition-4', + )) + # condition points to correct upstream output + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-6'] + .input_definitions.parameters[ + 'pipelinechannel--condition-branches-1-pipelinechannel--condition-branches-1-oneof-1'] + .parameter_type, type_utils.STRING) + # inner task consumes from condition input parameter + self.assertEqual( + roll_die_pipeline.pipeline_spec.components['comp-condition-6'].dag + .tasks['print-and-return-3'].inputs.parameters['text'] + .component_input_parameter, + 'pipelinechannel--condition-branches-1-pipelinechannel--condition-branches-1-oneof-1' + ) + + def test_oneof_in_fstring(self): + with self.assertRaisesRegex( + NotImplementedError, + f'dsl\.OneOf does not support string interpolation\.'): + + @dsl.pipeline + def roll_die_pipeline(): + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + with dsl.Else(): + t3 = print_and_return_with_output_key(text='Draw!') + print_and_return( + text=f"Final result: {dsl.OneOf(t1.output, t2.output, t3.outputs['output_key'])}" + ) + + def test_type_checking_parameters(self): + with self.assertRaisesRegex( + type_utils.InconsistentTypeException, + "Incompatible argument passed to the input 'val' of component 'print-int': Argument type 'STRING' is incompatible with the input type 'NUMBER_INTEGER'", + ): + + @dsl.component + def print_int(val: int): + print(val) + + @dsl.pipeline + def roll_die_pipeline(): + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + with dsl.Else(): + t3 = print_and_return_with_output_key(text='Draw!') + print_int( + val=dsl.OneOf(t1.output, t2.output, + t3.outputs['output_key'])) + + def test_oneof_of_oneof(self): + with self.assertRaisesRegex( + ValueError, + r'dsl.OneOf cannot be used inside of another dsl\.OneOf\.'): + + @dsl.pipeline + def roll_die_pipeline() -> str: + outer_flip_coin_task = flip_coin() + with dsl.If(outer_flip_coin_task.output == 'heads'): + inner_flip_coin_task = flip_coin() + with dsl.If(inner_flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Else(): + t2 = print_and_return(text='Got tails!') + t3 = dsl.OneOf(t1.output, t2.output) + with dsl.Else(): + t4 = print_and_return(text='First flip was not heads!') + return dsl.OneOf(t3, t4.output) + + +class TestPythonicArtifactAuthoring(unittest.TestCase): + # python component + def test_pythonic_input_artifact(self): + + @dsl.component + def pythonic_style(in_artifact: Artifact): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + 'system.Artifact', + ) + + self.assertFalse( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.parameters) + + @dsl.component + def standard_style(in_artifact: Input[Artifact]): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + standard_style.pipeline_spec.components['comp-standard-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + ) + + def test_pythonic_input_artifact_optional(self): + + @dsl.component + def pythonic_style(in_artifact: Optional[Artifact] = None): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + 'system.Artifact', + ) + + self.assertFalse( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.parameters) + + @dsl.component + def standard_style(in_artifact: Optional[Input[Artifact]] = None): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + standard_style.pipeline_spec.components['comp-standard-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + ) + + def test_pythonic_input_list_of_artifacts(self): + + @dsl.component + def pythonic_style(in_artifact: List[Artifact]): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + 'system.Artifact', + ) + self.assertTrue( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].is_artifact_list) + + self.assertFalse( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.parameters) + + @dsl.component + def standard_style(in_artifact: Input[List[Artifact]]): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + standard_style.pipeline_spec.components['comp-standard-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + ) + + def test_pythonic_input_list_of_artifacts_optional(self): + + @dsl.component + def pythonic_style(in_artifact: Optional[List[Artifact]] = None): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + 'system.Artifact', + ) + self.assertTrue( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].is_artifact_list) + + self.assertFalse( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.parameters) + + @dsl.component + def standard_style(in_artifact: Optional[Input[List[Artifact]]] = None): + print(in_artifact) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + standard_style.pipeline_spec.components['comp-standard-style'] + .input_definitions.artifacts['in_artifact'].artifact_type + .schema_title, + ) + + def test_pythonic_output_artifact(self): + + @dsl.component + def pythonic_style() -> Artifact: + return Artifact(uri='gs://my_bucket/foo') + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.artifacts['Output'].artifact_type.schema_title, + 'system.Artifact', + ) + + self.assertFalse( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.parameters) + + @dsl.component + def standard_style(named_output: Output[Artifact]): + return Artifact(uri='gs://my_bucket/foo') + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.artifacts['Output'].artifact_type.schema_title, + standard_style.pipeline_spec.components['comp-standard-style'] + .output_definitions.artifacts['named_output'].artifact_type + .schema_title, + ) + + def test_pythonic_output_artifact_multiple_returns(self): + + @dsl.component + def pythonic_style() -> NamedTuple('outputs', a=Artifact, d=Dataset): + a = Artifact(uri='gs://my_bucket/foo/artifact') + d = Artifact(uri='gs://my_bucket/foo/dataset') + outputs = NamedTuple('outputs', a=Artifact, d=Dataset) + return outputs(a=a, d=d) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.artifacts['a'].artifact_type.schema_title, + 'system.Artifact', + ) + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.artifacts['d'].artifact_type.schema_title, + 'system.Dataset', + ) + + self.assertFalse( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.parameters) + + @dsl.component + def standard_style(a: Output[Artifact], d: Output[Dataset]): + a.uri = 'gs://my_bucket/foo/artifact' + d.uri = 'gs://my_bucket/foo/dataset' + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.artifacts['a'].artifact_type.schema_title, + standard_style.pipeline_spec.components['comp-standard-style'] + .output_definitions.artifacts['a'].artifact_type.schema_title, + ) + + self.assertEqual( + pythonic_style.pipeline_spec.components['comp-pythonic-style'] + .output_definitions.artifacts['d'].artifact_type.schema_title, + standard_style.pipeline_spec.components['comp-standard-style'] + .output_definitions.artifacts['d'].artifact_type.schema_title, + ) + + def test_pythonic_output_list_artifacts(self): + + with self.assertRaisesRegex( + ValueError, + r"Output lists of artifacts are only supported for pipelines\. Got output list of artifacts for output parameter 'Output' of component 'pythonic-style'\." + ): + + @dsl.component + def pythonic_style() -> List[Artifact]: + pass + + def test_mixed_component_authoring_styles(self): + # can be permitted, since the expected behavior is unambiguous + + # in traditional; out pythonic + @dsl.component + def back_compat_style(in_artifact: Input[Artifact]) -> Artifact: + print(in_artifact) + return Artifact(uri='gs://my_bucket/foo') + + self.assertTrue(back_compat_style.pipeline_spec) + + # out traditional; in pythonic + @dsl.component + def mixed_style(in_artifact: Artifact, out_artifact: Output[Artifact]): + print(in_artifact) + out_artifact.uri = 'gs://my_bucket/foo' + + self.assertTrue(mixed_style.pipeline_spec) + + # pipeline + def test_pipeline_input_artifact(self): + + @dsl.component + def pythonic_style(in_artifact: Artifact): + print(in_artifact) + + @dsl.pipeline + def my_pipeline(in_artifact: Artifact): + pythonic_style(in_artifact=in_artifact) + + self.assertEqual( + my_pipeline.pipeline_spec.root.input_definitions + .artifacts['in_artifact'].artifact_type.schema_title, + 'system.Artifact', + ) + + self.assertFalse( + my_pipeline.pipeline_spec.root.input_definitions.parameters) + + def test_pipeline_input_artifact_optional(self): + + @dsl.component + def pythonic_style(in_artifact: Optional[Artifact] = None): + print(in_artifact) + + @dsl.pipeline + def my_pipeline(in_artifact: Optional[Artifact] = None): + pythonic_style(in_artifact=in_artifact) + + self.assertEqual( + my_pipeline.pipeline_spec.root.input_definitions + .artifacts['in_artifact'].artifact_type.schema_title, + 'system.Artifact', + ) + + self.assertFalse( + my_pipeline.pipeline_spec.root.input_definitions.parameters) + + def test_pipeline_input_list_of_artifacts(self): + + @dsl.component + def pythonic_style(in_artifact: List[Artifact]): + print(in_artifact) + + @dsl.pipeline + def my_pipeline(in_artifact: List[Artifact]): + pythonic_style(in_artifact=in_artifact) + + self.assertEqual( + my_pipeline.pipeline_spec.root.input_definitions + .artifacts['in_artifact'].artifact_type.schema_title, + 'system.Artifact', + ) + self.assertTrue(my_pipeline.pipeline_spec.root.input_definitions + .artifacts['in_artifact'].is_artifact_list) + + self.assertFalse( + my_pipeline.pipeline_spec.root.input_definitions.parameters) + + def test_pipeline_input_list_of_artifacts_optional(self): + + @dsl.component + def pythonic_style(in_artifact: Optional[List[Artifact]] = None): + print(in_artifact) + + @dsl.pipeline + def my_pipeline(in_artifact: Optional[List[Artifact]] = None): + pythonic_style(in_artifact=in_artifact) + + self.assertEqual( + my_pipeline.pipeline_spec.root.input_definitions + .artifacts['in_artifact'].artifact_type.schema_title, + 'system.Artifact', + ) + + self.assertFalse( + my_pipeline.pipeline_spec.root.input_definitions.parameters) + + def test_pipeline_output_artifact(self): + + @dsl.component + def pythonic_style() -> Artifact: + return Artifact(uri='gs://my_bucket/foo') + + @dsl.pipeline + def my_pipeline() -> Artifact: + return pythonic_style().output + + self.assertEqual( + my_pipeline.pipeline_spec.root.output_definitions + .artifacts['Output'].artifact_type.schema_title, 'system.Artifact') + + self.assertFalse( + my_pipeline.pipeline_spec.root.output_definitions.parameters) + + def test_pipeline_output_list_of_artifacts(self): + + @dsl.component + def noop() -> Artifact: + # write artifact + return Artifact(uri='gs://my_bucket/foo/bar') + + @dsl.pipeline + def my_pipeline() -> List[Artifact]: + with dsl.ParallelFor([1, 2, 3]): + t = noop() + + return dsl.Collected(t.output) + + self.assertEqual( + my_pipeline.pipeline_spec.root.output_definitions + .artifacts['Output'].artifact_type.schema_title, 'system.Artifact') + self.assertTrue(my_pipeline.pipeline_spec.root.output_definitions + .artifacts['Output'].is_artifact_list) + + self.assertFalse( + my_pipeline.pipeline_spec.root.output_definitions.parameters) + + # container + def test_container_input_artifact(self): + with self.assertRaisesRegex( + TypeError, + r"Container Components must wrap input and output artifact annotations with Input/Output type markers \(Input\[\] or Output\[\]\)\. Got function input 'in_artifact' with annotation \." + ): + + @dsl.container_component + def comp(in_artifact: Artifact): + return dsl.ContainerSpec(image='alpine', command=['pwd']) + + def test_container_input_artifact_optional(self): + with self.assertRaisesRegex( + TypeError, + r"Container Components must wrap input and output artifact annotations with Input/Output type markers \(Input\[\] or Output\[\]\)\. Got function input 'in_artifact' with annotation \." + ): + + @dsl.container_component + def comp(in_artifact: Optional[Artifact] = None): + return dsl.ContainerSpec(image='alpine', command=['pwd']) + + def test_container_input_list_of_artifacts(self): + with self.assertRaisesRegex( + TypeError, + r"Container Components must wrap input and output artifact annotations with Input/Output type markers \(Input\[\] or Output\[\]\)\. Got function input 'in_artifact' with annotation typing\.List\[kfp\.dsl\.types\.artifact_types\.Artifact\]\." + ): + + @dsl.container_component + def comp(in_artifact: List[Artifact]): + return dsl.ContainerSpec(image='alpine', command=['pwd']) + + def test_container_input_list_of_artifacts_optional(self): + with self.assertRaisesRegex( + TypeError, + r"Container Components must wrap input and output artifact annotations with Input/Output type markers \(Input\[\] or Output\[\]\)\. Got function input 'in_artifact' with annotation typing\.List\[kfp\.dsl\.types\.artifact_types\.Artifact\]\." + ): + + @dsl.container_component + def comp(in_artifact: Optional[List[Artifact]] = None): + return dsl.ContainerSpec(image='alpine', command=['pwd']) + + def test_container_output_artifact(self): + with self.assertRaisesRegex( + TypeError, + r'Return annotation should be either ContainerSpec or omitted for container components\.' + ): + + @dsl.container_component + def comp() -> Artifact: + return dsl.ContainerSpec(image='alpine', command=['pwd']) + + def test_container_output_list_of_artifact(self): + with self.assertRaisesRegex( + TypeError, + r'Return annotation should be either ContainerSpec or omitted for container components\.' + ): + + @dsl.container_component + def comp() -> List[Artifact]: + return dsl.ContainerSpec(image='alpine', command=['pwd']) + + +class TestPipelineSpecAttributeUniqueError(unittest.TestCase): + + def test_compiles(self): + # in a previous version of the KFP SDK there was an error when: + # - a component has a dsl.OutputPath parameter + # - the pipeline has an existing component by a different name + # - the user calls component.pipeline_spec inside their pipeline definition + # this was resolved coincidentally in + # https://github.com/kubeflow/pipelines/pull/10067, so test that it + # doesn't come back + + @dsl.container_component + def existing_comp(): + return dsl.ContainerSpec( + image='alpine', command=['echo'], args=['foo']) + + @dsl.container_component + def issue_comp(v: dsl.OutputPath(str)): + return dsl.ContainerSpec(image='alpine', command=['echo'], args=[v]) + + @dsl.pipeline + def my_pipeline(): + existing_comp() + issue_comp.pipeline_spec + + # should compile without error + self.assertTrue(my_pipeline.pipeline_spec) + + if __name__ == '__main__': unittest.main() diff --git a/sdk/python/kfp/compiler/compiler_utils.py b/sdk/python/kfp/compiler/compiler_utils.py index 1ae0c326bf..0bc525e250 100644 --- a/sdk/python/kfp/compiler/compiler_utils.py +++ b/sdk/python/kfp/compiler/compiler_utils.py @@ -14,14 +14,15 @@ """Utility methods for compiler implementation that is IR-agnostic.""" import collections -from copy import deepcopy +import copy from typing import DefaultDict, Dict, List, Mapping, Set, Tuple, Union -from kfp.components import for_loop -from kfp.components import pipeline_channel -from kfp.components import pipeline_context -from kfp.components import pipeline_task -from kfp.components import tasks_group +from kfp import dsl +from kfp.dsl import for_loop +from kfp.dsl import pipeline_channel +from kfp.dsl import pipeline_context +from kfp.dsl import pipeline_task +from kfp.dsl import tasks_group GroupOrTaskType = Union[tasks_group.TasksGroup, pipeline_task.PipelineTask] @@ -119,7 +120,18 @@ def _get_parent_groups_helper( return (tasks_to_groups, groups_to_groups) -# TODO: do we really need this? +def get_channels_from_condition( + operations: List[pipeline_channel.ConditionOperation], + collected_channels: list, +) -> None: + """Appends to collected_channels each pipeline channels used in each + operand of each operation in operations.""" + for operation in operations: + for operand in [operation.left_operand, operation.right_operand]: + if isinstance(operand, pipeline_channel.PipelineChannel): + collected_channels.append(operand) + + def get_condition_channels_for_tasks( root_group: tasks_group.TasksGroup, ) -> Mapping[str, Set[pipeline_channel.PipelineChannel]]: @@ -139,16 +151,13 @@ def _get_condition_channels_for_tasks_helper( current_conditions_channels, ): new_current_conditions_channels = current_conditions_channels - if isinstance(group, tasks_group.Condition): + if isinstance(group, tasks_group._ConditionBase): new_current_conditions_channels = list(current_conditions_channels) - if isinstance(group.condition.left_operand, - pipeline_channel.PipelineChannel): - new_current_conditions_channels.append( - group.condition.left_operand) - if isinstance(group.condition.right_operand, - pipeline_channel.PipelineChannel): - new_current_conditions_channels.append( - group.condition.right_operand) + get_channels_from_condition( + group.conditions, + new_current_conditions_channels, + ) + for task in group.tasks: for channel in new_current_conditions_channels: conditions[task.name].add(channel) @@ -250,10 +259,9 @@ def get_inputs_for_all_groups( if isinstance(channel_to_add, pipeline_channel.PipelineChannel): channels_to_add.append(channel_to_add) - if channel.task_name: + if channel.task: # The PipelineChannel is produced by a task. - - upstream_task = pipeline.tasks[channel.task_name] + upstream_task = channel.task upstream_groups, downstream_groups = ( _get_uncommon_ancestors( task_name_to_parent_groups=task_name_to_parent_groups, @@ -454,46 +462,116 @@ def get_outputs_for_all_groups( } outputs = collections.defaultdict(dict) - + processed_oneofs: Set[pipeline_channel.OneOfMixin] = set() # handle dsl.Collected consumed by tasks for task in pipeline.tasks.values(): for channel in task.channel_inputs: - if not isinstance(channel, for_loop.Collected): - continue - producer_task = pipeline.tasks[channel.task_name] - consumer_task = task - - upstream_groups, downstream_groups = ( - _get_uncommon_ancestors( - task_name_to_parent_groups=task_name_to_parent_groups, - group_name_to_parent_groups=group_name_to_parent_groups, - task1=producer_task, - task2=consumer_task, - )) - validate_parallel_for_fan_in_consumption_legal( - consumer_task_name=consumer_task.name, - upstream_groups=upstream_groups, - group_name_to_group=group_name_to_group, - ) + # TODO: migrate Collected to OneOfMixin style implementation, + # then simplify this logic to align with OneOfMixin logic + if isinstance(channel, dsl.Collected): + producer_task = pipeline.tasks[channel.task_name] + consumer_task = task + + upstream_groups, downstream_groups = ( + _get_uncommon_ancestors( + task_name_to_parent_groups=task_name_to_parent_groups, + group_name_to_parent_groups=group_name_to_parent_groups, + task1=producer_task, + task2=consumer_task, + )) + validate_parallel_for_fan_in_consumption_legal( + consumer_task_name=consumer_task.name, + upstream_groups=upstream_groups, + group_name_to_group=group_name_to_group, + ) + + # producer_task's immediate parent group and the name by which + # to surface the channel + surfaced_output_name = additional_input_name_for_pipeline_channel( + channel) + + # the highest-level task group that "consumes" the + # collected output + parent_consumer = downstream_groups[0] + producer_task_name = upstream_groups.pop() + + # process from the upstream groups from the inside out + for upstream_name in reversed(upstream_groups): + outputs[upstream_name][ + surfaced_output_name] = make_new_channel_for_collected_outputs( + channel_name=channel.name, + starting_channel=channel.output, + task_name=producer_task_name, + ) + + # on each iteration, mutate the channel being consumed so + # that it references the last parent group surfacer + channel.name = surfaced_output_name + channel.task_name = upstream_name + + # for the next iteration, set the consumer to the current + # surfacer (parent group) + producer_task_name = upstream_name - # producer_task's immediate parent group and the name by which - # to surface the channel + parent_of_current_surfacer = group_name_to_parent_groups[ + upstream_name][-2] + if parent_consumer in group_name_to_children[ + parent_of_current_surfacer]: + break + + elif isinstance(channel, pipeline_channel.OneOfMixin): + for inner_channel in channel.channels: + producer_task = pipeline.tasks[inner_channel.task_name] + consumer_task = task + upstream_groups, downstream_groups = ( + _get_uncommon_ancestors( + task_name_to_parent_groups=task_name_to_parent_groups, + group_name_to_parent_groups=group_name_to_parent_groups, + task1=producer_task, + task2=consumer_task, + )) + surfaced_output_name = additional_input_name_for_pipeline_channel( + inner_channel) + + # 1. get the oneof + # 2. find the task group that surfaced it + # 3. find the inner tasks reponsible + + for upstream_name in reversed(upstream_groups): + # skip the first task processed, since we don't need to add new outputs for the innermost task + if upstream_name == inner_channel.task.name: + continue + # # once we've hit the outermost condition-branches group, we're done + if upstream_name == channel.condition_branches_group.name: + outputs[upstream_name][channel.name] = channel + break + + # copy so we can update the inner channel for the next iteration + # use copy not deepcopy, since deepcopy will needlessly copy the entire pipeline + # this uses more memory than needed and some objects are uncopiable + outputs[upstream_name][ + surfaced_output_name] = copy.copy(inner_channel) + + inner_channel.name = surfaced_output_name + inner_channel.task_name = upstream_name + + processed_oneofs.add(channel) + + # handle dsl.Collected returned from pipeline + # TODO: consider migrating dsl.Collected returns to pattern used by dsl.OneOf, where the OneOf constructor returns a parameter/artifact channel, which fits in more cleanly into the existing compiler abtractions + for output_key, channel in pipeline_outputs_dict.items(): + if isinstance(channel, for_loop.Collected): surfaced_output_name = additional_input_name_for_pipeline_channel( channel) - - # the highest-level task group that "consumes" the - # collected output - parent_consumer = downstream_groups[0] + upstream_groups = task_name_to_parent_groups[channel.task_name][1:] producer_task_name = upstream_groups.pop() - - # process from the upstream groups from the inside out + # process upstream groups from the inside out, until getting to the pipeline level for upstream_name in reversed(upstream_groups): - outputs[upstream_name][ - surfaced_output_name] = make_new_channel_for_collected_outputs( - channel_name=channel.name, - starting_channel=channel.output, - task_name=producer_task_name, - ) + new_channel = make_new_channel_for_collected_outputs( + channel_name=channel.name, + starting_channel=channel.output, + task_name=producer_task_name, + ) # on each iteration, mutate the channel being consumed so # that it references the last parent group surfacer @@ -503,46 +581,46 @@ def get_outputs_for_all_groups( # for the next iteration, set the consumer to the current # surfacer (parent group) producer_task_name = upstream_name - - parent_of_current_surfacer = group_name_to_parent_groups[ - upstream_name][-2] - if parent_consumer in group_name_to_children[ - parent_of_current_surfacer]: - break - - # handle dsl.Collected returned from pipeline - for output_key, channel in pipeline_outputs_dict.items(): - if isinstance(channel, for_loop.Collected): - surfaced_output_name = additional_input_name_for_pipeline_channel( - channel) + outputs[upstream_name][surfaced_output_name] = new_channel + + # after surfacing from all inner TasksGroup, change the PipelineChannel output to also return from the correct TasksGroup + pipeline_outputs_dict[ + output_key] = make_new_channel_for_collected_outputs( + channel_name=surfaced_output_name, + starting_channel=channel.output, + task_name=upstream_name, + ) + elif isinstance(channel, pipeline_channel.OneOfMixin): + # if the output has already been consumed by a task before it is returned, we don't need to reprocess it + if channel in processed_oneofs: + continue + for inner_channel in channel.channels: + producer_task = pipeline.tasks[inner_channel.task_name] upstream_groups = task_name_to_parent_groups[ - channel.task_name][1:] - producer_task_name = upstream_groups.pop() - # process upstream groups from the inside out, until getting to the pipeline level - for upstream_name in reversed(upstream_groups): - new_channel = make_new_channel_for_collected_outputs( - channel_name=channel.name, - starting_channel=channel.output, - task_name=producer_task_name, - ) - - # on each iteration, mutate the channel being consumed so - # that it references the last parent group surfacer - channel.name = surfaced_output_name - channel.task_name = upstream_name + inner_channel.task_name][1:] + surfaced_output_name = additional_input_name_for_pipeline_channel( + inner_channel) - # for the next iteration, set the consumer to the current - # surfacer (parent group) - producer_task_name = upstream_name - outputs[upstream_name][surfaced_output_name] = new_channel - - # after surfacing from all inner TasksGroup, change the PipelineChannel output to also return from the correct TasksGroup - pipeline_outputs_dict[ - output_key] = make_new_channel_for_collected_outputs( - channel_name=surfaced_output_name, - starting_channel=channel.output, - task_name=upstream_name, - ) + # 1. get the oneof + # 2. find the task group that surfaced it + # 3. find the inner tasks reponsible + for upstream_name in reversed(upstream_groups): + # skip the first task processed, since we don't need to add new outputs for the innermost task + if upstream_name == inner_channel.task.name: + continue + # # once we've hit the outermost condition-branches group, we're done + if upstream_name == channel.condition_branches_group.name: + outputs[upstream_name][channel.name] = channel + break + + # copy so we can update the inner channel for the next iteration + # use copy not deepcopy, since deepcopy will needlessly copy the entire pipeline + # this uses more memory than needed and some objects are uncopiable + outputs[upstream_name][surfaced_output_name] = copy.copy( + inner_channel) + + inner_channel.name = surfaced_output_name + inner_channel.task_name = upstream_name return outputs, pipeline_outputs_dict @@ -625,22 +703,17 @@ def get_dependencies( """ dependencies = collections.defaultdict(set) for task in pipeline.tasks.values(): - upstream_task_names = set() + upstream_task_names: Set[Union[pipeline_task.PipelineTask, + tasks_group.TasksGroup]] = set() task_condition_inputs = list(condition_channels[task.name]) - for channel in task.channel_inputs + task_condition_inputs: - if channel.task_name: - upstream_task_names.add(channel.task_name) - upstream_task_names |= set(task.dependent_tasks) - - for upstream_task_name in upstream_task_names: - # the dependent op could be either a BaseOp or an opsgroup - if upstream_task_name in pipeline.tasks: - upstream_task = pipeline.tasks[upstream_task_name] - elif upstream_task_name in group_name_to_group: - upstream_task = group_name_to_group[upstream_task_name] - else: - raise ValueError( - f'Compiler cannot find task: {upstream_task_name}.') + all_channels = task.channel_inputs + task_condition_inputs + upstream_task_names.update( + {channel.task for channel in all_channels if channel.task}) + # dependent tasks is tasks on which .after was called and can only be the names of PipelineTasks, not TasksGroups + upstream_task_names.update( + {pipeline.tasks[after_task] for after_task in task.dependent_tasks}) + + for upstream_task in upstream_task_names: upstream_groups, downstream_groups = _get_uncommon_ancestors( task_name_to_parent_groups=task_name_to_parent_groups, @@ -650,7 +723,7 @@ def get_dependencies( ) # uncommon upstream ancestor check - uncommon_upstream_groups = deepcopy(upstream_groups) + uncommon_upstream_groups = copy.deepcopy(upstream_groups) uncommon_upstream_groups.remove( upstream_task.name ) # because a task's `upstream_groups` contains the task's name @@ -661,11 +734,14 @@ def get_dependencies( dependent_group = group_name_to_group.get( uncommon_upstream_groups[0], None) - if isinstance(dependent_group, - (tasks_group.Condition, tasks_group.ExitHandler)): + if isinstance( + dependent_group, + (tasks_group._ConditionBase, tasks_group.ExitHandler)): raise InvalidTopologyException( f'{ILLEGAL_CROSS_DAG_ERROR_PREFIX} A downstream task cannot depend on an upstream task within a dsl.{dependent_group.__class__.__name__} context unless the downstream is within that context too. Found task {task.name} which depends on upstream task {upstream_task.name} within an uncommon dsl.{dependent_group.__class__.__name__} context.' ) + # TODO: migrate Collected to OneOfMixin style implementation, + # then make this validation dsl.Collected-aware elif isinstance(dependent_group, tasks_group.ParallelFor): raise InvalidTopologyException( f'{ILLEGAL_CROSS_DAG_ERROR_PREFIX} A downstream task cannot depend on an upstream task within a dsl.{dependent_group.__class__.__name__} context unless the downstream is within that context too or the outputs are begin fanned-in to a list using dsl.{for_loop.Collected.__name__}. Found task {task.name} which depends on upstream task {upstream_task.name} within an uncommon dsl.{dependent_group.__class__.__name__} context.' diff --git a/sdk/python/kfp/compiler/compiler_utils_test.py b/sdk/python/kfp/compiler/compiler_utils_test.py index 61866fb740..ec20833b30 100644 --- a/sdk/python/kfp/compiler/compiler_utils_test.py +++ b/sdk/python/kfp/compiler/compiler_utils_test.py @@ -16,7 +16,7 @@ from absl.testing import parameterized from kfp.compiler import compiler_utils -from kfp.components import pipeline_channel +from kfp.dsl import pipeline_channel class TestAdditionalInputNameForPipelineChannel(parameterized.TestCase): diff --git a/sdk/python/kfp/compiler/pipeline_spec_builder.py b/sdk/python/kfp/compiler/pipeline_spec_builder.py index 67f9786dde..a7e5546c10 100644 --- a/sdk/python/kfp/compiler/pipeline_spec_builder.py +++ b/sdk/python/kfp/compiler/pipeline_spec_builder.py @@ -24,16 +24,17 @@ from google.protobuf import struct_pb2 import kfp from kfp.compiler import compiler_utils -from kfp.components import for_loop -from kfp.components import pipeline_channel -from kfp.components import pipeline_context -from kfp.components import pipeline_task -from kfp.components import placeholders -from kfp.components import structures -from kfp.components import tasks_group -from kfp.components import utils -from kfp.components.types import artifact_types -from kfp.components.types import type_utils +from kfp.dsl import component_factory +from kfp.dsl import for_loop +from kfp.dsl import pipeline_channel +from kfp.dsl import pipeline_context +from kfp.dsl import pipeline_task +from kfp.dsl import placeholders +from kfp.dsl import structures +from kfp.dsl import tasks_group +from kfp.dsl import utils +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_utils from kfp.pipeline_spec import pipeline_spec_pb2 import yaml @@ -45,8 +46,6 @@ tasks_group.TasksGroupType.EXIT_HANDLER: tasks_group.ExitHandler, } -_SINGLE_OUTPUT_NAME = 'Output' - def to_protobuf_value(value: type_utils.PARAMETER_TYPES) -> struct_pb2.Value: """Creates a google.protobuf.struct_pb2.Value message out of a provide @@ -129,11 +128,35 @@ def build_task_spec_for_task( task._task_spec.retry_policy.to_proto()) for input_name, input_value in task.inputs.items(): + # since LoopArgument and LoopArgumentVariable are narrower types than PipelineParameterChannel, start with it + if isinstance(input_value, for_loop.LoopArgument): + + component_input_parameter = ( + compiler_utils.additional_input_name_for_pipeline_channel( + input_value)) + assert component_input_parameter in parent_component_inputs.parameters, \ + f'component_input_parameter: {component_input_parameter} not found. All inputs: {parent_component_inputs}' + pipeline_task_spec.inputs.parameters[ + input_name].component_input_parameter = ( + component_input_parameter) + + elif isinstance(input_value, for_loop.LoopArgumentVariable): - if isinstance(input_value, - pipeline_channel.PipelineArtifactChannel) or ( - isinstance(input_value, for_loop.Collected) and - input_value.is_artifact_channel): + component_input_parameter = ( + compiler_utils.additional_input_name_for_pipeline_channel( + input_value.loop_argument)) + assert component_input_parameter in parent_component_inputs.parameters, \ + f'component_input_parameter: {component_input_parameter} not found. All inputs: {parent_component_inputs}' + pipeline_task_spec.inputs.parameters[ + input_name].component_input_parameter = ( + component_input_parameter) + pipeline_task_spec.inputs.parameters[ + input_name].parameter_expression_selector = ( + f'parseJson(string_value)["{input_value.subvar_name}"]') + elif isinstance(input_value, + pipeline_channel.PipelineArtifactChannel) or ( + isinstance(input_value, for_loop.Collected) and + input_value.is_artifact_channel): if input_value.task_name: # Value is produced by an upstream task. @@ -201,31 +224,6 @@ def build_task_spec_for_task( input_name].component_input_parameter = ( component_input_parameter) - elif isinstance(input_value, for_loop.LoopArgument): - - component_input_parameter = ( - compiler_utils.additional_input_name_for_pipeline_channel( - input_value)) - assert component_input_parameter in parent_component_inputs.parameters, \ - f'component_input_parameter: {component_input_parameter} not found. All inputs: {parent_component_inputs}' - pipeline_task_spec.inputs.parameters[ - input_name].component_input_parameter = ( - component_input_parameter) - - elif isinstance(input_value, for_loop.LoopArgumentVariable): - - component_input_parameter = ( - compiler_utils.additional_input_name_for_pipeline_channel( - input_value.loop_argument)) - assert component_input_parameter in parent_component_inputs.parameters, \ - f'component_input_parameter: {component_input_parameter} not found. All inputs: {parent_component_inputs}' - pipeline_task_spec.inputs.parameters[ - input_name].component_input_parameter = ( - component_input_parameter) - pipeline_task_spec.inputs.parameters[ - input_name].parameter_expression_selector = ( - f'parseJson(string_value)["{input_value.subvar_name}"]') - elif isinstance(input_value, str): # Handle extra input due to string concat pipeline_channels = ( @@ -302,11 +300,6 @@ def build_task_spec_for_task( 'str, int, float, bool, dict, and list.' f'Got {input_value} of type {type(input_value)}.') - if task._ignore_upstream_failure_tag: - pipeline_task_spec.trigger_policy.strategy = ( - pipeline_spec_pb2.PipelineTaskSpec.TriggerPolicy.TriggerStrategy - .ALL_UPSTREAM_TASKS_COMPLETED) - return pipeline_task_spec @@ -340,7 +333,8 @@ def build_component_spec_for_task( """ for input_name, input_spec in (task.component_spec.inputs or {}).items(): if not is_exit_task and type_utils.is_task_final_status_type( - input_spec.type) and not is_compiled_component: + input_spec.type + ) and not is_compiled_component and not task._ignore_upstream_failure_tag: raise ValueError( f'PipelineTaskFinalStatus can only be used in an exit task. Parameter {input_name} of a non exit task has type PipelineTaskFinalStatus.' ) @@ -422,12 +416,13 @@ def _build_component_spec_from_component_spec_structure( return component_spec -def _connect_dag_outputs( +def connect_single_dag_output( component_spec: pipeline_spec_pb2.ComponentSpec, output_name: str, output_channel: pipeline_channel.PipelineChannel, ) -> None: - """Connects dag output to a subtask output. + """Connects a DAG output to a subtask output when the subtask output + contains only one channel (i.e., not OneOfMixin). Args: component_spec: The component spec to modify its dag outputs. @@ -456,14 +451,71 @@ def _connect_dag_outputs( output_name].value_from_parameter.output_parameter_key = output_channel.name +def connect_oneof_dag_output( + component_spec: pipeline_spec_pb2.ComponentSpec, + output_name: str, + oneof_output: pipeline_channel.OneOfMixin, +) -> None: + """Connects a output to the OneOf output returned by the DAG's internal + condition-branches group. + + Args: + component_spec: The component spec to modify its DAG outputs. + output_name: The name of the DAG output. + oneof_output: The OneOfMixin object returned by the pipeline (OneOf in user code). + """ + if isinstance(oneof_output, pipeline_channel.OneOfArtifact): + if output_name not in component_spec.output_definitions.artifacts: + raise ValueError( + f'Pipeline or component output not defined: {output_name}. You may be missing a type annotation.' + ) + for channel in oneof_output.channels: + component_spec.dag.outputs.artifacts[ + output_name].artifact_selectors.append( + pipeline_spec_pb2.DagOutputsSpec.ArtifactSelectorSpec( + producer_subtask=channel.task_name, + output_artifact_key=channel.name, + )) + if isinstance(oneof_output, pipeline_channel.OneOfParameter): + if output_name not in component_spec.output_definitions.parameters: + raise ValueError( + f'Pipeline or component output not defined: {output_name}. You may be missing a type annotation.' + ) + for channel in oneof_output.channels: + component_spec.dag.outputs.parameters[ + output_name].value_from_oneof.parameter_selectors.append( + pipeline_spec_pb2.DagOutputsSpec.ParameterSelectorSpec( + producer_subtask=channel.task_name, + output_parameter_key=channel.name, + )) + + def _build_dag_outputs( component_spec: pipeline_spec_pb2.ComponentSpec, dag_outputs: Dict[str, pipeline_channel.PipelineChannel], ) -> None: - """Builds DAG output spec.""" + """Connects the DAG's outputs to a TaskGroup's ComponentSpec and validates + it is present in the component interface. + + Args: + component_spec: The ComponentSpec. + dag_outputs: Dictionary of output key to output channel. + """ for output_name, output_channel in dag_outputs.items(): - _connect_dag_outputs(component_spec, output_name, output_channel) - # Valid dag outputs covers all outptus in component definition. + if not isinstance(output_channel, pipeline_channel.PipelineChannel): + raise ValueError( + f"Got unknown pipeline output '{output_name}' of type {output_channel}." + ) + connect_single_dag_output(component_spec, output_name, output_channel) + + validate_dag_outputs(component_spec) + + +def validate_dag_outputs( + component_spec: pipeline_spec_pb2.ComponentSpec) -> None: + """Validates the DAG's ComponentSpec specifies the source task for all of + its ComponentSpec inputs (input_definitions) and outputs + (output_definitions).""" for output_name in component_spec.output_definitions.artifacts: if output_name not in component_spec.dag.outputs.artifacts: raise ValueError(f'Missing pipeline output: {output_name}.') @@ -472,6 +524,31 @@ def _build_dag_outputs( raise ValueError(f'Missing pipeline output: {output_name}.') +def build_oneof_dag_outputs( + component_spec: pipeline_spec_pb2.ComponentSpec, + oneof_outputs: Dict[str, pipeline_channel.OneOfMixin], +) -> None: + """Connects the DAG's OneOf outputs to a TaskGroup's ComponentSpec and + validates it is present in the component interface. + + Args: + component_spec: The ComponentSpec. + oneof_outputs: Dictionary of output key to OneOf output channel. + """ + for output_name, oneof_output in oneof_outputs.items(): + for channel in oneof_output.channels: + if not isinstance(channel, pipeline_channel.PipelineChannel): + raise ValueError( + f"Got unknown pipeline output '{output_name}' of type {type(channel)}." + ) + connect_oneof_dag_output( + component_spec, + output_name, + oneof_output, + ) + validate_dag_outputs(component_spec) + + def build_importer_spec_for_task( task: pipeline_task.PipelineTask ) -> pipeline_spec_pb2.PipelineDeploymentConfig.ImporterSpec: @@ -494,7 +571,7 @@ def build_importer_spec_for_task( importer_spec.metadata.CopyFrom(metadata_protobuf_struct) if isinstance(task.importer_spec.artifact_uri, - pipeline_channel.PipelineParameterChannel): + pipeline_channel.PipelineChannel): importer_spec.artifact_uri.runtime_parameter = 'uri' elif isinstance(task.importer_spec.artifact_uri, str): importer_spec.artifact_uri.constant.string_value = task.importer_spec.artifact_uri @@ -631,7 +708,7 @@ def build_component_spec_for_group( else: component_spec.output_definitions.parameters[ output_name].parameter_type = type_utils.get_parameter_type( - channel.channel_type) + output.channel_type) return component_spec @@ -709,22 +786,38 @@ def _update_task_spec_for_loop_group( input_name=pipeline_task_spec.parameter_iterator.item_input) -def _resolve_condition_operands( - left_operand: Union[str, pipeline_channel.PipelineChannel], - right_operand: Union[str, pipeline_channel.PipelineChannel], -) -> Tuple[str, str]: - """Resolves values and PipelineChannels for condition operands. +def _binary_operations_to_cel_conjunctive( + operations: List[pipeline_channel.ConditionOperation]) -> str: + """Converts a list of ConditionOperation to a CEL string with placeholders. + Each ConditionOperation will be joined the others via the conjunctive (&&). + + Args: + operations: The binary operations to convert to convert and join. + + Returns: + The binary operations as a CEL string. + """ + operands = [ + _single_binary_operation_to_cel_condition(operation=bin_op) + for bin_op in operations + ] + return ' && '.join(operands) + + +def _single_binary_operation_to_cel_condition( + operation: pipeline_channel.ConditionOperation) -> str: + """Converts a ConditionOperation to a CEL string with placeholders. Args: - left_operand: The left operand of a condition expression. - right_operand: The right operand of a condition expression. + operation: The binary operation to convert to a string. Returns: - A tuple of the resolved operands values: - (left_operand_value, right_operand_value). + The binary operation as a CEL string. """ + left_operand = operation.left_operand + right_operand = operation.right_operand - # Pre-scan the operand to get the type of constant value if there's any. + # cannot make comparisons involving particular types for value_or_reference in [left_operand, right_operand]: if isinstance(value_or_reference, pipeline_channel.PipelineChannel): parameter_type = type_utils.get_parameter_type( @@ -738,8 +831,10 @@ def _resolve_condition_operands( input_name = compiler_utils.additional_input_name_for_pipeline_channel( value_or_reference) raise ValueError( - f'Conditional requires scalar parameter values for comparison. Found input "{input_name}" of type {value_or_reference.channel_type} in pipeline definition instead.' + f'Conditional requires primitive parameter values for comparison. Found input "{input_name}" of type {value_or_reference.channel_type} in pipeline definition instead.' ) + + # ensure the types compared are the same or compatible parameter_types = set() for value_or_reference in [left_operand, right_operand]: if isinstance(value_or_reference, pipeline_channel.PipelineChannel): @@ -822,11 +917,16 @@ def _resolve_condition_operands( operand_values.append(operand_value) - return tuple(operand_values) + left_operand_value, right_operand_value = tuple(operand_values) + + condition_string = ( + f'{left_operand_value} {operation.operator} {right_operand_value}') + + return f'!({condition_string})' if operation.negate else condition_string def _update_task_spec_for_condition_group( - group: tasks_group.Condition, + group: tasks_group._ConditionBase, pipeline_task_spec: pipeline_spec_pb2.PipelineTaskSpec, ) -> None: """Updates PipelineTaskSpec for condition group. @@ -835,15 +935,9 @@ def _update_task_spec_for_condition_group( group: The condition group to update task spec for. pipeline_task_spec: The pipeline task spec to update in place. """ - left_operand_value, right_operand_value = _resolve_condition_operands( - group.condition.left_operand, group.condition.right_operand) - - condition_string = ( - f'{left_operand_value} {group.condition.operator} {right_operand_value}' - ) + condition = _binary_operations_to_cel_conjunctive(group.conditions) pipeline_task_spec.trigger_policy.CopyFrom( - pipeline_spec_pb2.PipelineTaskSpec.TriggerPolicy( - condition=condition_string)) + pipeline_spec_pb2.PipelineTaskSpec.TriggerPolicy(condition=condition)) def build_task_spec_for_exit_task( @@ -954,7 +1048,7 @@ def build_task_spec_for_group( group=group, pipeline_task_spec=pipeline_task_spec, ) - elif isinstance(group, tasks_group.Condition): + elif isinstance(group, tasks_group._ConditionBase): _update_task_spec_for_condition_group( group=group, pipeline_task_spec=pipeline_task_spec, @@ -1236,17 +1330,14 @@ def build_spec_by_group( _build_dag_outputs(subgroup_component_spec, subgroup_output_channels) - elif isinstance(subgroup, tasks_group.Condition): + elif isinstance(subgroup, tasks_group._ConditionBase): # "Punch the hole", adding inputs needed by its subgroups or # tasks. condition_subgroup_channels = list(subgroup_input_channels) - for operand in [ - subgroup.condition.left_operand, - subgroup.condition.right_operand, - ]: - if isinstance(operand, pipeline_channel.PipelineChannel): - condition_subgroup_channels.append(operand) + + compiler_utils.get_channels_from_condition( + subgroup.conditions, condition_subgroup_channels) subgroup_component_spec = build_component_spec_for_group( input_pipeline_channels=condition_subgroup_channels, @@ -1277,6 +1368,23 @@ def build_spec_by_group( is_parent_component_root=is_parent_component_root, ) + # handles the conditional group wrapping only + elif isinstance(subgroup, tasks_group.ConditionBranches): + subgroup_component_spec = build_component_spec_for_group( + input_pipeline_channels=subgroup_input_channels, + output_pipeline_channels=subgroup_output_channels, + ) + + subgroup_task_spec = build_task_spec_for_group( + group=subgroup, + pipeline_channels=subgroup_input_channels, + tasks_in_current_dag=tasks_in_current_dag, + is_parent_component_root=is_parent_component_root, + ) + # oneof is the only type of output a ConditionBranches group can have + build_oneof_dag_outputs(subgroup_component_spec, + subgroup_output_channels) + else: raise RuntimeError( f'Unexpected task/group type: Got {subgroup} of type ' @@ -1289,6 +1397,11 @@ def build_spec_by_group( subgroup_task_spec.dependent_tasks.extend( [utils.sanitize_task_name(dep) for dep in group_dependencies]) + # Modify the task inputs for PipelineTaskFinalStatus if ignore_upstream_failure is used + # Must be done after dependencies are added + if isinstance(subgroup, pipeline_task.PipelineTask): + modify_task_for_ignore_upstream_failure( + task=subgroup, pipeline_task_spec=subgroup_task_spec) # Add component spec subgroup_component_name = utils.make_name_unique_by_adding_index( name=subgroup_component_name, @@ -1315,6 +1428,42 @@ def build_spec_by_group( ) +def modify_task_for_ignore_upstream_failure( + task: pipeline_task.PipelineTask, + pipeline_task_spec: pipeline_spec_pb2.PipelineTaskSpec, +): + if task._ignore_upstream_failure_tag: + pipeline_task_spec.trigger_policy.strategy = ( + pipeline_spec_pb2.PipelineTaskSpec.TriggerPolicy.TriggerStrategy + .ALL_UPSTREAM_TASKS_COMPLETED) + + for input_name, input_spec in (task.component_spec.inputs or + {}).items(): + if not type_utils.is_task_final_status_type(input_spec.type): + continue + + if len(pipeline_task_spec.dependent_tasks) == 0: + if task.parent_task_group.group_type == tasks_group.TasksGroupType.PIPELINE: + raise compiler_utils.InvalidTopologyException( + f"Tasks that use '.ignore_upstream_failure()' and 'PipelineTaskFinalStatus' must have exactly one dependent upstream task. Got task '{pipeline_task_spec.task_info.name} with no upstream dependencies." + ) + else: + # TODO: permit additional PipelineTaskFinalStatus flexibility by "punching the hole" through Condition and ParallelFor groups + raise compiler_utils.InvalidTopologyException( + f"Tasks that use '.ignore_upstream_failure()' and 'PipelineTaskFinalStatus' must have exactly one dependent upstream task within the same control flow scope. Got task '{pipeline_task_spec.task_info.name}' beneath a 'dsl.{group_type_to_dsl_class[task.parent_task_group.group_type].__name__}' that does not also contain the upstream dependent task." + ) + + # if >1 dependent task, ambiguous to which upstream task the PipelineTaskFinalStatus should correspond, since there is no ExitHandler that bundles these together + if len(pipeline_task_spec.dependent_tasks) > 1: + raise compiler_utils.InvalidTopologyException( + f"Tasks that use '.ignore_upstream_failure()' and 'PipelineTaskFinalStatus' must have exactly one dependent upstream task. Got {len(pipeline_task_spec.dependent_tasks)} dependent tasks: {pipeline_task_spec.dependent_tasks}." + ) + + pipeline_task_spec.inputs.parameters[ + input_name].task_final_status.producer_task = pipeline_task_spec.dependent_tasks[ + 0] + + def platform_config_to_platform_spec( platform_config: dict, executor_label: str, @@ -1630,6 +1779,28 @@ def _rename_component_refs( old_name_to_new_name[old_component_name]].CopyFrom(component_spec) +def validate_pipeline_outputs_dict( + pipeline_outputs_dict: Dict[str, pipeline_channel.PipelineChannel]): + for channel in pipeline_outputs_dict.values(): + if isinstance(channel, for_loop.Collected): + # this validation doesn't apply to Collected + continue + + elif isinstance(channel, pipeline_channel.OneOfMixin): + if channel.condition_branches_group.parent_task_group.group_type != tasks_group.TasksGroupType.PIPELINE: + raise compiler_utils.InvalidTopologyException( + f'Pipeline outputs may only be returned from the top level of the pipeline function scope. Got pipeline output dsl.{pipeline_channel.OneOf.__name__} from within the control flow group dsl.{channel.condition_branches_group.parent_task_group.__class__.__name__}.' + ) + + elif isinstance(channel, pipeline_channel.PipelineChannel): + if channel.task.parent_task_group.group_type != tasks_group.TasksGroupType.PIPELINE: + raise compiler_utils.InvalidTopologyException( + f'Pipeline outputs may only be returned from the top level of the pipeline function scope. Got pipeline output from within the control flow group dsl.{channel.task.parent_task_group.__class__.__name__}.' + ) + else: + raise ValueError(f'Got unknown pipeline output: {channel}.') + + def create_pipeline_spec( pipeline: pipeline_context.Pipeline, component_spec: structures.ComponentSpec, @@ -1665,6 +1836,8 @@ def create_pipeline_spec( # an output from a task in a condition group, for example, which isn't # caught until submission time using Vertex SDK client pipeline_outputs_dict = convert_pipeline_outputs_to_dict(pipeline_outputs) + validate_pipeline_outputs_dict(pipeline_outputs_dict) + root_group = pipeline.groups[0] all_groups = compiler_utils.get_all_groups(root_group) @@ -1760,7 +1933,7 @@ def convert_pipeline_outputs_to_dict( if pipeline_outputs is None: return {} elif isinstance(pipeline_outputs, pipeline_channel.PipelineChannel): - return {_SINGLE_OUTPUT_NAME: pipeline_outputs} + return {component_factory.SINGLE_OUTPUT_NAME: pipeline_outputs} elif isinstance(pipeline_outputs, tuple) and hasattr( pipeline_outputs, '_asdict'): return dict(pipeline_outputs._asdict()) @@ -1770,7 +1943,7 @@ def convert_pipeline_outputs_to_dict( def write_pipeline_spec_to_file( pipeline_spec: pipeline_spec_pb2.PipelineSpec, - pipeline_description: str, + pipeline_description: Union[str, None], platform_spec: pipeline_spec_pb2.PlatformSpec, package_path: str, ) -> None: diff --git a/sdk/python/kfp/compiler/read_write_test.py b/sdk/python/kfp/compiler/read_write_test.py index 8ecdb30c4f..9be9d4ca2c 100644 --- a/sdk/python/kfp/compiler/read_write_test.py +++ b/sdk/python/kfp/compiler/read_write_test.py @@ -22,9 +22,9 @@ from absl.testing import parameterized from kfp import compiler from kfp import components -from kfp.components import placeholders -from kfp.components import python_component -from kfp.components import structures +from kfp.dsl import placeholders +from kfp.dsl import python_component +from kfp.dsl import structures import yaml _PROJECT_ROOT = os.path.abspath(os.path.join(__file__, *([os.path.pardir] * 5))) @@ -146,7 +146,8 @@ def _test_serialization_deserialization_consistency(self, yaml_file: str): original_component) self.assertEqual( handle_expected_diffs(original_component.component_spec), - handle_expected_diffs(reloaded_component.component_spec)) + handle_expected_diffs(reloaded_component.component_spec), + f'\n\n\nError with (de)serialization consistency of: {yaml_file}') def _test_serialization_correctness( self, @@ -158,7 +159,8 @@ def _test_serialization_correctness( pipeline = import_obj_from_file(python_file, function_name) compiled_result = self._compile_and_read_yaml(pipeline) golden_result = load_compiled_file(yaml_file) - self.assertEqual(compiled_result, golden_result) + self.assertEqual(compiled_result, golden_result, + f'\n\n\nError with compiling: {python_file}') @parameterized.parameters(create_test_cases()) def test( @@ -173,12 +175,12 @@ def test( """Tests serialization and deserialization consistency and correctness. Args: - name (str): '{test_group_name}-{test_case_name}'. Useful for print statements/debugging. - test_case (str): Test case name (without file extension). - test_data_dir (str): The directory containing the test case files. - function (str, optional): The function name to compile. - read (bool): Whether the pipeline/component supports deserialization from YAML (IR, except for V1 component YAML back compatability tests). - write (bool): Whether the pipeline/component supports compilation from a Python file. + name: '{test_group_name}-{test_case_name}'. Useful for print statements/debugging. + test_case: Test case name (without file extension). + test_data_dir: The directory containing the test case files. + function: The function name to compile. + read: Whether the pipeline/component supports deserialization from YAML (IR, except for V1 component YAML back compatability tests). + write: Whether the pipeline/component supports compilation from a Python file. """ yaml_file = os.path.join(test_data_dir, f'{test_case}.yaml') py_file = os.path.join(test_data_dir, f'{test_case}.py') diff --git a/sdk/python/kfp/components/__init__.py b/sdk/python/kfp/components/__init__.py index 3f183b6267..01eb072e12 100644 --- a/sdk/python/kfp/components/__init__.py +++ b/sdk/python/kfp/components/__init__.py @@ -24,10 +24,13 @@ 'YamlComponent', ] -from kfp.components.base_component import BaseComponent -from kfp.components.container_component import ContainerComponent -from kfp.components.python_component import PythonComponent -from kfp.components.yaml_component import load_component_from_file -from kfp.components.yaml_component import load_component_from_text -from kfp.components.yaml_component import load_component_from_url -from kfp.components.yaml_component import YamlComponent +from kfp.components.load_yaml_utilities import load_component_from_file +from kfp.components.load_yaml_utilities import load_component_from_text +from kfp.components.load_yaml_utilities import load_component_from_url +# keep this for backward compatibility with user code "from kfp.components import placholders" and similar +from kfp.dsl import base_component # noqa: keep unused import +from kfp.dsl import placeholders # noqa: keep unused import +from kfp.dsl.base_component import BaseComponent +from kfp.dsl.container_component_class import ContainerComponent +from kfp.dsl.python_component import PythonComponent +from kfp.dsl.yaml_component import YamlComponent diff --git a/sdk/python/kfp/components/component_factory_test.py b/sdk/python/kfp/components/component_factory_test.py deleted file mode 100644 index 6cfd5b5c8d..0000000000 --- a/sdk/python/kfp/components/component_factory_test.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2022 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List -import unittest - -from kfp import dsl -from kfp.components import component_factory -from kfp.components import structures -from kfp.components.component_decorator import component -from kfp.components.types.artifact_types import Artifact -from kfp.components.types.artifact_types import Model -from kfp.components.types.type_annotations import OutputPath -from kfp.dsl import Input -from kfp.dsl import Output - - -class TestGetPackagesToInstallCommand(unittest.TestCase): - - def test_with_no_packages_to_install(self): - packages_to_install = [] - - command = component_factory._get_packages_to_install_command( - packages_to_install) - self.assertEqual(command, []) - - def test_with_packages_to_install_and_no_pip_index_url(self): - packages_to_install = ['package1', 'package2'] - - command = component_factory._get_packages_to_install_command( - packages_to_install) - concat_command = ' '.join(command) - for package in packages_to_install: - self.assertTrue(package in concat_command) - - def test_with_packages_to_install_with_pip_index_url(self): - packages_to_install = ['package1', 'package2'] - pip_index_urls = ['https://myurl.org/simple'] - - command = component_factory._get_packages_to_install_command( - packages_to_install, pip_index_urls) - concat_command = ' '.join(command) - for package in packages_to_install + pip_index_urls: - self.assertTrue(package in concat_command) - - -class TestInvalidParameterName(unittest.TestCase): - - def test_output_named_Output(self): - - with self.assertRaisesRegex(ValueError, - r'"Output" is an invalid parameter name.'): - - @component - def comp(Output: OutputPath(str)): - pass - - def test_output_named_Output_with_string_output(self): - - with self.assertRaisesRegex(ValueError, - r'"Output" is an invalid parameter name.'): - - @component - def comp(Output: OutputPath(str), text: str) -> str: - pass - - -class TestExtractComponentInterfaceListofArtifacts(unittest.TestCase): - - def test_python_component_input(self): - - def comp(i: Input[List[Model]]): - ... - - component_spec = component_factory.extract_component_interface(comp) - self.assertEqual(component_spec.name, 'comp') - self.assertEqual(component_spec.description, None) - self.assertEqual( - component_spec.inputs, { - 'i': - structures.InputSpec( - type='system.Model@0.0.1', - default=None, - is_artifact_list=True) - }) - - def test_custom_container_component_input(self): - - def comp(i: Input[List[Artifact]]): - ... - - component_spec = component_factory.extract_component_interface( - comp, containerized=True) - self.assertEqual(component_spec.name, 'comp') - self.assertEqual(component_spec.description, None) - self.assertEqual( - component_spec.inputs, { - 'i': - structures.InputSpec( - type='system.Artifact@0.0.1', - default=None, - is_artifact_list=True) - }) - - def test_pipeline_input(self): - - def comp(i: Input[List[Model]]): - ... - - component_spec = component_factory.extract_component_interface(comp) - self.assertEqual(component_spec.name, 'comp') - self.assertEqual(component_spec.description, None) - self.assertEqual( - component_spec.inputs, { - 'i': - structures.InputSpec( - type='system.Model@0.0.1', - default=None, - is_artifact_list=True) - }) - - -class TestArtifactStringInInputpathOutputpath(unittest.TestCase): - - def test_unknown(self): - - @dsl.component - def comp( - i: dsl.InputPath('MyCustomType'), - o: dsl.OutputPath('MyCustomType'), - ): - ... - - self.assertEqual(comp.component_spec.outputs['o'].type, - 'system.Artifact@0.0.1') - self.assertFalse(comp.component_spec.outputs['o'].is_artifact_list) - self.assertEqual(comp.component_spec.inputs['i'].type, - 'system.Artifact@0.0.1') - self.assertFalse(comp.component_spec.inputs['i'].is_artifact_list) - - def test_known_v1_back_compat(self): - - @dsl.component - def comp( - i: dsl.InputPath('Dataset'), - o: dsl.OutputPath('Dataset'), - ): - ... - - self.assertEqual(comp.component_spec.outputs['o'].type, - 'system.Dataset@0.0.1') - self.assertFalse(comp.component_spec.outputs['o'].is_artifact_list) - self.assertEqual(comp.component_spec.inputs['i'].type, - 'system.Dataset@0.0.1') - self.assertFalse(comp.component_spec.inputs['i'].is_artifact_list) - - -class TestOutputListsOfArtifactsTemporarilyBlocked(unittest.TestCase): - - def test_python_component(self): - with self.assertRaisesRegex( - ValueError, - r"Output lists of artifacts are only supported for pipelines\. Got output list of artifacts for output parameter 'output_list' of component 'comp'\." - ): - - @dsl.component - def comp(output_list: Output[List[Artifact]]): - ... - - def test_container_component(self): - with self.assertRaisesRegex( - ValueError, - r"Output lists of artifacts are only supported for pipelines\. Got output list of artifacts for output parameter 'output_list' of component 'comp'\." - ): - - @dsl.container_component - def comp(output_list: Output[List[Artifact]]): - return dsl.ContainerSpec(image='alpine') - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/components/executor.py b/sdk/python/kfp/components/executor.py deleted file mode 100644 index 0f43790999..0000000000 --- a/sdk/python/kfp/components/executor.py +++ /dev/null @@ -1,368 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import inspect -import json -import os -from typing import Any, Callable, Dict, List, Optional, Union - -from kfp.components import python_component -from kfp.components import task_final_status -from kfp.components.types import artifact_types -from kfp.components.types import type_annotations - - -class Executor(): - """Executor executes v2-based Python function components.""" - - def __init__(self, executor_input: Dict, - function_to_execute: Union[Callable, - python_component.PythonComponent]): - if hasattr(function_to_execute, 'python_func'): - self._func = function_to_execute.python_func - else: - self._func = function_to_execute - - self._input = executor_input - self._input_artifacts: Dict[str, - Union[artifact_types.Artifact, - List[artifact_types.Artifact]]] = {} - self._output_artifacts: Dict[str, artifact_types.Artifact] = {} - - for name, artifacts in self._input.get('inputs', - {}).get('artifacts', {}).items(): - list_of_artifact_proto_structs = artifacts.get('artifacts') - if list_of_artifact_proto_structs: - annotation = self._func.__annotations__[name] - # InputPath has no attribute __origin__ and also should be handled as a single artifact - if type_annotations.is_Input_Output_artifact_annotation( - annotation) and type_annotations.is_list_of_artifacts( - annotation.__origin__): - self._input_artifacts[name] = [ - self.make_artifact( - msg, - name, - self._func, - ) for msg in list_of_artifact_proto_structs - ] - else: - self._input_artifacts[name] = self.make_artifact( - list_of_artifact_proto_structs[0], - name, - self._func, - ) - - for name, artifacts in self._input.get('outputs', - {}).get('artifacts', {}).items(): - list_of_artifact_proto_structs = artifacts.get('artifacts') - if list_of_artifact_proto_structs: - output_artifact = self.make_artifact( - list_of_artifact_proto_structs[0], - name, - self._func, - ) - self._output_artifacts[name] = output_artifact - self.makedirs_recursively(output_artifact.path) - - self._return_annotation = inspect.signature( - self._func).return_annotation - self._executor_output = {} - - def make_artifact( - self, - runtime_artifact: Dict, - name: str, - func: Callable, - ) -> Any: - annotation = func.__annotations__.get(name) - if isinstance(annotation, type_annotations.InputPath): - schema_title, _ = annotation.type.split('@') - if schema_title in artifact_types._SCHEMA_TITLE_TO_TYPE: - artifact_cls = artifact_types._SCHEMA_TITLE_TO_TYPE[ - schema_title] - else: - raise TypeError( - f'Invalid type argument to {type_annotations.InputPath.__name__}: {annotation.type}' - ) - else: - artifact_cls = annotation - return create_artifact_instance( - runtime_artifact, artifact_cls=artifact_cls) - - def makedirs_recursively(self, path: str) -> None: - os.makedirs(os.path.dirname(path), exist_ok=True) - - def _get_input_artifact(self, name: str): - return self._input_artifacts.get(name) - - def _get_output_artifact(self, name: str): - return self._output_artifacts.get(name) - - def _get_input_parameter_value(self, parameter_name: str): - parameter_values = self._input.get('inputs', - {}).get('parameterValues', None) - - if parameter_values is not None: - return parameter_values.get(parameter_name, None) - - return None - - def _get_output_parameter_path(self, parameter_name: str): - parameter = self._input.get('outputs', - {}).get('parameters', - {}).get(parameter_name, None) - if parameter is None: - return None - - import os - path = parameter.get('outputFile', None) - if path: - os.makedirs(os.path.dirname(path), exist_ok=True) - return path - - def _get_output_artifact_path(self, artifact_name: str): - output_artifact = self._output_artifacts.get(artifact_name) - if not output_artifact: - raise ValueError( - f'Failed to get output artifact path for artifact name {artifact_name}' - ) - return output_artifact.path - - def _get_input_artifact_path(self, artifact_name: str): - input_artifact = self._input_artifacts.get(artifact_name) - if not input_artifact: - raise ValueError( - f'Failed to get input artifact path for artifact name {artifact_name}' - ) - return input_artifact.path - - def _write_output_parameter_value(self, name: str, - value: Union[str, int, float, bool, dict, - list, Dict, List]): - if isinstance(value, (float, int)): - output = str(value) - elif isinstance(value, str): - # value is already a string. - output = value - elif isinstance(value, (bool, list, dict)): - output = json.dumps(value) - else: - raise ValueError( - f'Unable to serialize unknown type `{value}` for parameter input with value `{type(value)}`' - ) - - if not self._executor_output.get('parameterValues'): - self._executor_output['parameterValues'] = {} - - self._executor_output['parameterValues'][name] = value - - def _write_output_artifact_payload(self, name: str, value: Any): - path = self._get_output_artifact_path(name) - with open(path, 'w') as f: - f.write(str(value)) - - # TODO: extract to a util - @classmethod - def _get_short_type_name(cls, type_name: str) -> str: - """Extracts the short form type name. - - This method is used for looking up serializer for a given type. - - For example: - typing.List -> List - typing.List[int] -> List - typing.Dict[str, str] -> Dict - List -> List - str -> str - - Args: - type_name: The original type name. - - Returns: - The short form type name or the original name if pattern doesn't match. - """ - import re - match = re.match('(typing\.)?(?P\w+)(?:\[.+\])?', type_name) - return match.group('type') if match else type_name - - # TODO: merge with type_utils.is_parameter_type - @classmethod - def _is_parameter(cls, annotation: Any) -> bool: - if type(annotation) == type: - return annotation in [str, int, float, bool, dict, list] - - # Annotation could be, for instance `typing.Dict[str, str]`, etc. - return cls._get_short_type_name(str(annotation)) in ['Dict', 'List'] - - @classmethod - def _is_artifact(cls, annotation: Any) -> bool: - if type(annotation) == type: - return type_annotations.is_artifact_class(annotation) - return False - - @classmethod - def _is_named_tuple(cls, annotation: Any) -> bool: - if type(annotation) == type: - return issubclass(annotation, tuple) and hasattr( - annotation, '_fields') and hasattr(annotation, - '__annotations__') - return False - - def _handle_single_return_value(self, output_name: str, - annotation_type: Any, return_value: Any): - if self._is_parameter(annotation_type): - origin_type = getattr(annotation_type, '__origin__', - None) or annotation_type - # relax float-typed return to allow both int and float. - if origin_type == float: - accepted_types = (int, float) - # TODO: relax str-typed return to allow all primitive types? - else: - accepted_types = origin_type - if not isinstance(return_value, accepted_types): - raise ValueError( - f'Function `{self._func.__name__}` returned value of type {type(return_value)}; want type {origin_type}' - ) - self._write_output_parameter_value(output_name, return_value) - elif self._is_artifact(annotation_type): - self._write_output_artifact_payload(output_name, return_value) - else: - raise RuntimeError( - f'Unknown return type: {annotation_type}. Must be one of the supported data types: https://www.kubeflow.org/docs/components/pipelines/v2/data-types/' - ) - - def _write_executor_output(self, func_output: Optional[Any] = None): - if self._output_artifacts: - self._executor_output['artifacts'] = {} - - for name, artifact in self._output_artifacts.items(): - runtime_artifact = { - 'name': artifact.name, - 'uri': artifact.uri, - 'metadata': artifact.metadata, - } - artifacts_list = {'artifacts': [runtime_artifact]} - - self._executor_output['artifacts'][name] = artifacts_list - - if func_output is not None: - if self._is_parameter(self._return_annotation) or self._is_artifact( - self._return_annotation): - # Note: single output is named `Output` in component.yaml. - self._handle_single_return_value('Output', - self._return_annotation, - func_output) - elif self._is_named_tuple(self._return_annotation): - if len(self._return_annotation._fields) != len(func_output): - raise RuntimeError( - f'Expected {len(self._return_annotation._fields)} return values from function `{self._func.__name__}`, got {len(func_output)}' - ) - for i in range(len(self._return_annotation._fields)): - field = self._return_annotation._fields[i] - field_type = self._return_annotation.__annotations__[field] - if type(func_output) == tuple: - field_value = func_output[i] - else: - field_value = getattr(func_output, field) - self._handle_single_return_value(field, field_type, - field_value) - else: - raise RuntimeError( - f'Unknown return type: {self._return_annotation}. Must be one of `str`, `int`, `float`, a subclass of `Artifact`, or a NamedTuple collection of these types.' - ) - - # This check is to ensure only one worker (in a mirrored, distributed training/compute strategy) attempts to write to the same executor output file at the same time using gcsfuse, which enforces immutability of files. - write_file = True - - CLUSTER_SPEC_ENV_VAR_NAME = 'CLUSTER_SPEC' - cluster_spec_string = os.environ.get(CLUSTER_SPEC_ENV_VAR_NAME) - if cluster_spec_string: - cluster_spec = json.loads(cluster_spec_string) - CHIEF_NODE_LABELS = {'workerpool0', 'chief', 'master'} - write_file = cluster_spec['task']['type'] in CHIEF_NODE_LABELS - - if write_file: - executor_output_path = self._input['outputs']['outputFile'] - os.makedirs(os.path.dirname(executor_output_path), exist_ok=True) - with open(executor_output_path, 'w') as f: - f.write(json.dumps(self._executor_output)) - - def execute(self): - annotations = inspect.getfullargspec(self._func).annotations - - # Function arguments. - func_kwargs = {} - - for k, v in annotations.items(): - if k == 'return': - continue - - # Annotations for parameter types could be written as, for example, - # `Optional[str]`. In this case, we need to strip off the part - # `Optional[]` to get the actual parameter type. - v = type_annotations.maybe_strip_optional_from_annotation(v) - - if v == task_final_status.PipelineTaskFinalStatus: - value = self._get_input_parameter_value(k) - func_kwargs[k] = task_final_status.PipelineTaskFinalStatus( - state=value.get('state'), - pipeline_job_resource_name=value.get( - 'pipelineJobResourceName'), - pipeline_task_name=value.get('pipelineTaskName'), - error_code=value.get('error').get('code', None), - error_message=value.get('error').get('message', None), - ) - - elif self._is_parameter(v): - value = self._get_input_parameter_value(k) - if value is not None: - func_kwargs[k] = value - - elif type_annotations.is_Input_Output_artifact_annotation(v): - if type_annotations.is_input_artifact(v): - func_kwargs[k] = self._get_input_artifact(k) - if type_annotations.is_output_artifact(v): - func_kwargs[k] = self._get_output_artifact(k) - - elif isinstance(v, type_annotations.OutputPath): - if self._is_parameter(v.type): - func_kwargs[k] = self._get_output_parameter_path(k) - else: - func_kwargs[k] = self._get_output_artifact_path(k) - - elif isinstance(v, type_annotations.InputPath): - func_kwargs[k] = self._get_input_artifact_path(k) - - result = self._func(**func_kwargs) - self._write_executor_output(result) - - -def create_artifact_instance( - runtime_artifact: Dict, - artifact_cls=artifact_types.Artifact, -) -> type: - """Creates an artifact class instances from a runtime artifact - dictionary.""" - schema_title = runtime_artifact.get('type', {}).get('schemaTitle', '') - - artifact_cls = artifact_types._SCHEMA_TITLE_TO_TYPE.get( - schema_title, artifact_cls) - return artifact_cls._from_executor_fields( - uri=runtime_artifact.get('uri', ''), - name=runtime_artifact.get('name', ''), - metadata=runtime_artifact.get('metadata', {}), - ) if hasattr(artifact_cls, '_from_executor_fields') else artifact_cls( - uri=runtime_artifact.get('uri', ''), - name=runtime_artifact.get('name', ''), - metadata=runtime_artifact.get('metadata', {}), - ) diff --git a/sdk/python/kfp/components/yaml_component.py b/sdk/python/kfp/components/load_yaml_utilities.py similarity index 59% rename from sdk/python/kfp/components/yaml_component.py rename to sdk/python/kfp/components/load_yaml_utilities.py index e60907c6ef..34342d3b0b 100644 --- a/sdk/python/kfp/components/yaml_component.py +++ b/sdk/python/kfp/components/load_yaml_utilities.py @@ -15,49 +15,12 @@ from typing import Optional, Tuple -from google.protobuf import json_format -from kfp import components -from kfp.components import structures -from kfp.pipeline_spec import pipeline_spec_pb2 +from kfp.dsl import structures +from kfp.dsl import yaml_component import requests -class YamlComponent(components.BaseComponent): - """A component loaded from a YAML file. - - **Note:** ``YamlComponent`` is not intended to be used to construct components directly. Use ``kfp.components.load_component_from_*()`` instead. - - Attribute: - component_spec: Component definition. - component_yaml: The yaml string that this component is loaded from. - """ - - def __init__( - self, - component_spec: structures.ComponentSpec, - component_yaml: str, - ): - super().__init__(component_spec=component_spec) - self.component_yaml = component_yaml - - @property - def pipeline_spec(self) -> pipeline_spec_pb2.PipelineSpec: - """Returns the pipeline spec of the component.""" - component_dict = structures.load_documents_from_yaml( - self.component_yaml)[0] - is_v1 = 'implementation' in set(component_dict.keys()) - if is_v1: - return self.component_spec.to_pipeline_spec() - else: - return json_format.ParseDict(component_dict, - pipeline_spec_pb2.PipelineSpec()) - - def execute(self, *args, **kwargs): - """Not implemented.""" - raise NotImplementedError - - -def load_component_from_text(text: str) -> YamlComponent: +def load_component_from_text(text: str) -> yaml_component.YamlComponent: """Loads a component from text. Args: @@ -66,12 +29,12 @@ def load_component_from_text(text: str) -> YamlComponent: Returns: Component loaded from YAML. """ - return YamlComponent( + return yaml_component.YamlComponent( component_spec=structures.ComponentSpec.from_yaml_documents(text), component_yaml=text) -def load_component_from_file(file_path: str) -> YamlComponent: +def load_component_from_file(file_path: str) -> yaml_component.YamlComponent: """Loads a component from a file. Args: @@ -91,9 +54,9 @@ def load_component_from_file(file_path: str) -> YamlComponent: return load_component_from_text(component_stream.read()) -def load_component_from_url(url: str, - auth: Optional[Tuple[str, - str]] = None) -> YamlComponent: +def load_component_from_url( + url: str, + auth: Optional[Tuple[str, str]] = None) -> yaml_component.YamlComponent: """Loads a component from a URL. Args: diff --git a/sdk/python/kfp/components/yaml_component_test.py b/sdk/python/kfp/components/load_yaml_utilities_test.py similarity index 91% rename from sdk/python/kfp/components/yaml_component_test.py rename to sdk/python/kfp/components/load_yaml_utilities_test.py index 3e56e3ce79..55ba29cf57 100644 --- a/sdk/python/kfp/components/yaml_component_test.py +++ b/sdk/python/kfp/components/load_yaml_utilities_test.py @@ -11,15 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for kfp.components.yaml_component.""" +"""Tests for kfp.dsl.yaml_component.""" import os import tempfile import textwrap import unittest -from kfp.components import structures -from kfp.components import yaml_component +from kfp import components +from kfp.dsl import structures SAMPLE_YAML = textwrap.dedent("""\ components: @@ -84,10 +84,10 @@ ] -class YamlComponentTest(unittest.TestCase): +class LoadYamlTests(unittest.TestCase): def test_load_component_from_text(self): - component = yaml_component.load_component_from_text(SAMPLE_YAML) + component = components.load_component_from_text(SAMPLE_YAML) self.assertEqual(component.component_spec.name, 'component-1') self.assertEqual(component.component_spec.outputs, {'output1': structures.OutputSpec(type='String')}) @@ -101,7 +101,7 @@ def test_load_component_from_file(self): path = os.path.join(tmpdir, 'sample_yaml.yaml') with open(path, 'w') as f: f.write(SAMPLE_YAML) - component = yaml_component.load_component_from_file(path) + component = components.load_component_from_file(path) self.assertEqual(component.component_spec.name, 'component-1') self.assertEqual(component.component_spec.outputs, {'output1': structures.OutputSpec(type='String')}) @@ -112,7 +112,7 @@ def test_load_component_from_file(self): def test_load_component_from_url(self): component_url = 'https://raw.githubusercontent.com/kubeflow/pipelines/7b49eadf621a9054e1f1315c86f95fb8cf8c17c3/sdk/python/kfp/compiler/test_data/components/identity.yaml' - component = yaml_component.load_component_from_url(component_url) + component = components.load_component_from_url(component_url) self.assertEqual(component.component_spec.name, 'identity') self.assertEqual(component.component_spec.outputs, diff --git a/sdk/python/kfp/components/pipeline_channel_test.py b/sdk/python/kfp/components/pipeline_channel_test.py deleted file mode 100644 index 060fe4ad23..0000000000 --- a/sdk/python/kfp/components/pipeline_channel_test.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.components.pipeline_channel.""" - -import unittest - -from absl.testing import parameterized -from kfp import dsl -from kfp.components import pipeline_channel - - -class PipelineChannelTest(parameterized.TestCase): - - def test_instantiate_pipline_channel(self): - with self.assertRaisesRegex( - TypeError, "Can't instantiate abstract class PipelineChannel"): - p = pipeline_channel.PipelineChannel( - name='channel', - channel_type='String', - ) - - def test_invalid_name(self): - with self.assertRaisesRegex( - ValueError, - 'Only letters, numbers, spaces, "_", and "-" are allowed in the ' - 'name. Must begin with a letter. Got name: 123_abc'): - p = pipeline_channel.create_pipeline_channel( - name='123_abc', - channel_type='String', - ) - - def test_task_name_and_value_both_set(self): - with self.assertRaisesRegex(ValueError, - 'task_name and value cannot be both set.'): - p = pipeline_channel.create_pipeline_channel( - name='abc', - channel_type='Integer', - task_name='task1', - value=123, - ) - - def test_invalid_type(self): - with self.assertRaisesRegex(TypeError, - 'Artifact is not a parameter type.'): - p = pipeline_channel.PipelineParameterChannel( - name='channel1', - channel_type='Artifact', - ) - - with self.assertRaisesRegex(TypeError, - 'String is not an artifact type.'): - p = pipeline_channel.PipelineArtifactChannel( - name='channel1', - channel_type='String', - task_name='task1', - is_artifact_list=False, - ) - - @parameterized.parameters( - { - 'pipeline_channel': - pipeline_channel.create_pipeline_channel( - name='channel1', - task_name='task1', - channel_type='String', - ), - 'str_repr': - '{{channel:task=task1;name=channel1;type=String;}}', - }, - { - 'pipeline_channel': - pipeline_channel.create_pipeline_channel( - name='channel2', - channel_type='Integer', - ), - 'str_repr': - '{{channel:task=;name=channel2;type=Integer;}}', - }, - { - 'pipeline_channel': - pipeline_channel.create_pipeline_channel( - name='channel3', - channel_type={'type_a': { - 'property_b': 'c' - }}, - task_name='task3', - ), - 'str_repr': - '{{channel:task=task3;name=channel3;type={"type_a": {"property_b": "c"}};}}', - }, - { - 'pipeline_channel': - pipeline_channel.create_pipeline_channel( - name='channel4', - channel_type='Float', - value=1.23, - ), - 'str_repr': - '{{channel:task=;name=channel4;type=Float;}}', - }, - { - 'pipeline_channel': - pipeline_channel.create_pipeline_channel( - name='channel5', - channel_type='system.Artifact@0.0.1', - task_name='task5', - ), - 'str_repr': - '{{channel:task=task5;name=channel5;type=system.Artifact@0.0.1;}}', - }, - ) - def test_str_repr(self, pipeline_channel, str_repr): - self.assertEqual(str_repr, str(pipeline_channel)) - - def test_extract_pipeline_channels(self): - p1 = pipeline_channel.create_pipeline_channel( - name='channel1', - channel_type='String', - value='abc', - ) - p2 = pipeline_channel.create_pipeline_channel( - name='channel2', - channel_type='customized_type_b', - task_name='task2', - ) - p3 = pipeline_channel.create_pipeline_channel( - name='channel3', - channel_type={'customized_type_c': { - 'property_c': 'value_c' - }}, - task_name='task3', - ) - stuff_chars = ' between ' - payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3) - params = pipeline_channel.extract_pipeline_channels_from_string(payload) - self.assertListEqual([p1, p2, p3], params) - - # Expecting the extract_pipelineparam_from_any to dedup pipeline channels - # among all the payloads. - payload = [ - str(p1) + stuff_chars + str(p2), - str(p2) + stuff_chars + str(p3) - ] - params = pipeline_channel.extract_pipeline_channels_from_any(payload) - self.assertListEqual([p1, p2, p3], params) - - -class TestCanAccessTask(unittest.TestCase): - - def test(self): - - @dsl.component - def comp() -> str: - return 'text' - - @dsl.pipeline - def my_pipeline(): - op1 = comp() - self.assertEqual(op1.output.task, op1) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/components/tasks_group.py b/sdk/python/kfp/components/tasks_group.py deleted file mode 100644 index c5390c6899..0000000000 --- a/sdk/python/kfp/components/tasks_group.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Definition for TasksGroup.""" - -import enum -from typing import Optional, Union - -from kfp.components import for_loop -from kfp.components import pipeline_channel -from kfp.components import pipeline_context -from kfp.components import pipeline_task - - -class TasksGroupType(str, enum.Enum): - """Types of TasksGroup.""" - PIPELINE = 'pipeline' - CONDITION = 'condition' - FOR_LOOP = 'for-loop' - EXIT_HANDLER = 'exit-handler' - - -class TasksGroup: - """Represents a logical group of tasks and groups of TasksGroups. - - This class is the base class for groups of tasks, such as tasks - sharing an exit handler, a condition branch, or a loop. This class - is not supposed to be used by pipeline authors. It is useful for - implementing a compiler. - - Attributes: - group_type: The type of the TasksGroup. - tasks: A list of all PipelineTasks in this group. - groups: A list of TasksGroups in this group. - display_name: The optional user given name of the group. - dependencies: A list of tasks or groups this group depends on. - is_root: If TasksGroup is root group. - """ - - def __init__( - self, - group_type: TasksGroupType, - name: Optional[str] = None, - is_root: bool = False, - ): - """Create a new instance of TasksGroup. - - Args: - group_type: The type of the group. - name: The name of the group. Used as display name in UI. - """ - self.group_type = group_type - self.tasks = [] - self.groups = [] - self.display_name = name - self.dependencies = [] - self.is_root = is_root - - def __enter__(self): - if not pipeline_context.Pipeline.get_default_pipeline(): - raise ValueError('Default pipeline not defined.') - - self._make_name_unique() - - pipeline_context.Pipeline.get_default_pipeline().push_tasks_group(self) - return self - - def __exit__(self, *unused_args): - pipeline_context.Pipeline.get_default_pipeline().pop_tasks_group() - - def _make_name_unique(self): - """Generates a unique TasksGroup name in the pipeline.""" - if not pipeline_context.Pipeline.get_default_pipeline(): - raise ValueError('Default pipeline not defined.') - - group_id = pipeline_context.Pipeline.get_default_pipeline( - ).get_next_group_id() - self.name = f'{self.group_type.value}-{group_id}' - self.name = self.name.replace('_', '-') - - def remove_task_recursive(self, task: pipeline_task.PipelineTask): - """Removes a task from the group recursively.""" - if self.tasks and task in self.tasks: - self.tasks.remove(task) - for group in self.groups or []: - group.remove_task_recursive(task) - - -class ExitHandler(TasksGroup): - """A class for setting an exit handler task that is invoked upon exiting a - group of other tasks. - - Args: - exit_task: The task that is invoked after exiting a group of other tasks. - name: The name of the exit handler group. - - Example: - :: - - exit_task = ExitComponent(...) - with ExitHandler(exit_task): - task1 = my_component1(...) - task2 = my_component2(...) - """ - - def __init__( - self, - exit_task: pipeline_task.PipelineTask, - name: Optional[str] = None, - ): - """Initializes a Condition task group.""" - super().__init__( - group_type=TasksGroupType.EXIT_HANDLER, - name=name, - is_root=False, - ) - - if exit_task.dependent_tasks: - raise ValueError('exit_task cannot depend on any other tasks.') - - # Removing exit_task form any group - pipeline_context.Pipeline.get_default_pipeline( - ).remove_task_from_groups(exit_task) - - # Set is_exit_handler since the compiler might be using this attribute. - exit_task.is_exit_handler = True - - self.exit_task = exit_task - - -class Condition(TasksGroup): - """A class for creating conditional control flow within a pipeline - definition. - - Args: - condition: A comparative expression that evaluates to True or False. At least one of the operands must be an output from an upstream task or a pipeline parameter. - name: The name of the condition group. - - Example: - :: - - task1 = my_component1(...) - with Condition(task1.output=='pizza', 'pizza-condition'): - task2 = my_component2(...) - """ - - def __init__( - self, - condition: pipeline_channel.ConditionOperator, - name: Optional[str] = None, - ): - """Initializes a conditional task group.""" - super().__init__( - group_type=TasksGroupType.CONDITION, - name=name, - is_root=False, - ) - self.condition = condition - - -class ParallelFor(TasksGroup): - """A class for creating parallelized for loop control flow over a static - set of items within a pipeline definition. - - Args: - items: The items to loop over. It can be either a constant Python list or a list output from an upstream task. - name: The name of the for loop group. - parallelism: The maximum number of concurrent iterations that can be scheduled for execution. A value of 0 represents unconstrained parallelism (default is unconstrained). - - Example: - :: - - with dsl.ParallelFor( - items=[{'a': 1, 'b': 10}, {'a': 2, 'b': 20}], - parallelism=1 - ) as item: - task1 = my_component(..., number=item.a) - task2 = my_component(..., number=item.b) - - In the example, the group of tasks containing ``task1`` and ``task2`` would - be executed twice, once with case ``args=[{'a': 1, 'b': 10}]`` and once with - case ``args=[{'a': 2, 'b': 20}]``. The ``parallelism=1`` setting causes only - 1 execution to be scheduled at a time. - """ - - def __init__( - self, - items: Union[for_loop.ItemList, pipeline_channel.PipelineChannel], - name: Optional[str] = None, - parallelism: Optional[int] = None, - ): - """Initializes a for loop task group.""" - parallelism = parallelism or 0 - if parallelism < 0: - raise ValueError( - f'ParallelFor parallelism must be >= 0. Got: {parallelism}.') - - super().__init__( - group_type=TasksGroupType.FOR_LOOP, - name=name, - is_root=False, - ) - - if isinstance(items, pipeline_channel.PipelineChannel): - self.loop_argument = for_loop.LoopArgument.from_pipeline_channel( - items) - self.items_is_pipeline_channel = True - else: - self.loop_argument = for_loop.LoopArgument.from_raw_items( - raw_items=items, - name_code=pipeline_context.Pipeline.get_default_pipeline() - .get_next_group_id(), - ) - self.items_is_pipeline_channel = False - - self.parallelism_limit = parallelism - - def __enter__(self) -> for_loop.LoopArgument: - super().__enter__() - return self.loop_argument diff --git a/sdk/python/kfp/components/test_data/simple_yaml.yaml b/sdk/python/kfp/components/test_data/simple_yaml.yaml deleted file mode 100644 index 8db8477f7b..0000000000 --- a/sdk/python/kfp/components/test_data/simple_yaml.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: component_1 -inputs: - input1: {type: String} -outputs: - output1: {type: String} -implementation: - container: - image: alpine - command: - - sh - - -c - - 'set -ex - - echo "$0" > "$1"' - - {inputValue: input1} - - {outputPath: output1} \ No newline at end of file diff --git a/sdk/python/kfp/components/types/artifact_types_test.py b/sdk/python/kfp/components/types/artifact_types_test.py deleted file mode 100644 index 517d5f9b4e..0000000000 --- a/sdk/python/kfp/components/types/artifact_types_test.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.components.types.artifact_types.""" - -import json -import os -import unittest - -from absl.testing import parameterized -from kfp.components.types import artifact_types - - -class ArtifactsTest(parameterized.TestCase): - - def test_complex_metrics(self): - metrics = artifact_types.ClassificationMetrics() - metrics.log_roc_data_point(threshold=0.1, tpr=98.2, fpr=96.2) - metrics.log_roc_data_point(threshold=24.3, tpr=24.5, fpr=98.4) - metrics.set_confusion_matrix_categories(['dog', 'cat', 'horses']) - metrics.log_confusion_matrix_row('dog', [2, 6, 0]) - metrics.log_confusion_matrix_cell('cat', 'dog', 3) - metrics.log_confusion_matrix_cell('horses', 'horses', 3) - metrics.metadata['test'] = 1.0 - with open( - os.path.join( - os.path.dirname(__file__), 'test_data', - 'expected_io_types_classification_metrics.json') - ) as json_file: - expected_json = json.load(json_file) - self.assertEqual(expected_json, metrics.metadata) - - def test_complex_metrics_bulk_loading(self): - metrics = artifact_types.ClassificationMetrics() - metrics.log_roc_curve( - fpr=[85.1, 85.1, 85.1], - tpr=[52.6, 52.6, 52.6], - threshold=[53.6, 53.6, 53.6]) - metrics.log_confusion_matrix(['dog', 'cat', 'horses'], - [[2, 6, 0], [3, 5, 6], [5, 7, 8]]) - with open( - os.path.join( - os.path.dirname(__file__), 'test_data', - 'expected_io_types_bulk_load_classification_metrics.json') - ) as json_file: - expected_json = json.load(json_file) - self.assertEqual(expected_json, metrics.metadata) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/components/v1_components.py b/sdk/python/kfp/components/v1_components.py deleted file mode 100644 index f276fa9702..0000000000 --- a/sdk/python/kfp/components/v1_components.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2018-2022 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import warnings - -from kfp.components import v1_structures -import yaml - - -def _load_component_spec_from_component_text( - text) -> v1_structures.ComponentSpec: - component_dict = yaml.safe_load(text) - component_spec = v1_structures.ComponentSpec.from_dict(component_dict) - - if isinstance(component_spec.implementation, - v1_structures.ContainerImplementation) and ( - component_spec.implementation.container.command is None): - warnings.warn( - 'Container component must specify command to be compatible with KFP ' - 'v2 compatible mode and emissary executor, which will be the default' - ' executor for KFP v2.' - 'https://www.kubeflow.org/docs/components/pipelines/installation/choose-executor/', - category=FutureWarning, - ) - - # Calculating hash digest for the component - data = text if isinstance(text, bytes) else text.encode('utf-8') - data = data.replace(b'\r\n', b'\n') # Normalizing line endings - digest = hashlib.sha256(data).hexdigest() - component_spec._digest = digest - - return component_spec diff --git a/sdk/python/kfp/dsl/__init__.py b/sdk/python/kfp/dsl/__init__.py index a70190ca78..d2372156c7 100644 --- a/sdk/python/kfp/dsl/__init__.py +++ b/sdk/python/kfp/dsl/__init__.py @@ -14,31 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# runtime dependencies __all__ = [ - 'component', - 'container_component', - 'pipeline', - 'importer', - 'ContainerSpec', - 'Condition', - 'ExitHandler', - 'ParallelFor', - 'Collected', 'Input', 'Output', 'InputPath', 'OutputPath', - 'IfPresentPlaceholder', - 'ConcatPlaceholder', 'PipelineTaskFinalStatus', - 'PIPELINE_JOB_NAME_PLACEHOLDER', - 'PIPELINE_JOB_RESOURCE_NAME_PLACEHOLDER', - 'PIPELINE_JOB_ID_PLACEHOLDER', - 'PIPELINE_TASK_NAME_PLACEHOLDER', - 'PIPELINE_TASK_ID_PLACEHOLDER', - 'PIPELINE_ROOT_PLACEHOLDER', - 'PIPELINE_JOB_CREATE_TIME_UTC_PLACEHOLDER', - 'PIPELINE_JOB_SCHEDULE_TIME_UTC_PLACEHOLDER', 'Artifact', 'ClassificationMetrics', 'Dataset', @@ -47,8 +29,32 @@ 'Metrics', 'Model', 'SlicedClassificationMetrics', - 'PipelineTask', + 'get_uri', + 'PIPELINE_JOB_NAME_PLACEHOLDER', + 'PIPELINE_JOB_RESOURCE_NAME_PLACEHOLDER', + 'PIPELINE_JOB_ID_PLACEHOLDER', + 'PIPELINE_TASK_NAME_PLACEHOLDER', + 'PIPELINE_TASK_ID_PLACEHOLDER', + 'PIPELINE_ROOT_PLACEHOLDER', + 'PIPELINE_JOB_CREATE_TIME_UTC_PLACEHOLDER', + 'PIPELINE_JOB_SCHEDULE_TIME_UTC_PLACEHOLDER', ] +import os + +from kfp.dsl.task_final_status import PipelineTaskFinalStatus +from kfp.dsl.types.artifact_types import Artifact +from kfp.dsl.types.artifact_types import ClassificationMetrics +from kfp.dsl.types.artifact_types import Dataset +from kfp.dsl.types.artifact_types import get_uri +from kfp.dsl.types.artifact_types import HTML +from kfp.dsl.types.artifact_types import Markdown +from kfp.dsl.types.artifact_types import Metrics +from kfp.dsl.types.artifact_types import Model +from kfp.dsl.types.artifact_types import SlicedClassificationMetrics +from kfp.dsl.types.type_annotations import InputAnnotation +from kfp.dsl.types.type_annotations import InputPath +from kfp.dsl.types.type_annotations import OutputAnnotation +from kfp.dsl.types.type_annotations import OutputPath try: from typing import Annotated @@ -57,34 +63,7 @@ from typing import TypeVar -from kfp.components.component_decorator import component -from kfp.components.container_component_decorator import container_component -from kfp.components.for_loop import Collected -from kfp.components.importer_node import importer -from kfp.components.pipeline_context import pipeline -from kfp.components.pipeline_task import PipelineTask -from kfp.components.placeholders import ConcatPlaceholder -from kfp.components.placeholders import IfPresentPlaceholder -from kfp.components.structures import ContainerSpec -from kfp.components.task_final_status import PipelineTaskFinalStatus -from kfp.components.tasks_group import Condition -from kfp.components.tasks_group import ExitHandler -from kfp.components.tasks_group import ParallelFor -from kfp.components.types.artifact_types import Artifact -from kfp.components.types.artifact_types import ClassificationMetrics -from kfp.components.types.artifact_types import Dataset -from kfp.components.types.artifact_types import HTML -from kfp.components.types.artifact_types import Markdown -from kfp.components.types.artifact_types import Metrics -from kfp.components.types.artifact_types import Model -from kfp.components.types.artifact_types import SlicedClassificationMetrics -from kfp.components.types.type_annotations import InputAnnotation -from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import OutputAnnotation -from kfp.components.types.type_annotations import OutputPath - # hack: constants and custom type generics have to be defined here to be captured by autodoc and autodocsumm used in ./docs/conf.py - PIPELINE_JOB_NAME_PLACEHOLDER = '{{$.pipeline_job_name}}' """A placeholder used to obtain a pipeline job name within a task at pipeline runtime. @@ -247,3 +226,41 @@ def my_pipeline(): producer_task = artifact_producer() artifact_consumer(model=producer_task.output) """ + +# compile-time only dependencies +if os.environ.get('_KFP_RUNTIME', 'false') != 'true': + from kfp.dsl.component_decorator import component + from kfp.dsl.container_component_decorator import container_component + # TODO: Collected should be moved to pipeline_channel.py, consistent with OneOf + from kfp.dsl.for_loop import Collected + from kfp.dsl.importer_node import importer + from kfp.dsl.pipeline_channel import OneOf + from kfp.dsl.pipeline_context import pipeline + from kfp.dsl.pipeline_task import PipelineTask + from kfp.dsl.placeholders import ConcatPlaceholder + from kfp.dsl.placeholders import IfPresentPlaceholder + from kfp.dsl.structures import ContainerSpec + from kfp.dsl.tasks_group import Condition + from kfp.dsl.tasks_group import Elif + from kfp.dsl.tasks_group import Else + from kfp.dsl.tasks_group import ExitHandler + from kfp.dsl.tasks_group import If + from kfp.dsl.tasks_group import ParallelFor + __all__.extend([ + 'component', + 'container_component', + 'pipeline', + 'importer', + 'ContainerSpec', + 'Condition', + 'If', + 'Elif', + 'Else', + 'OneOf', + 'ExitHandler', + 'ParallelFor', + 'Collected', + 'IfPresentPlaceholder', + 'ConcatPlaceholder', + 'PipelineTask', + ]) diff --git a/sdk/python/kfp/components/base_component.py b/sdk/python/kfp/dsl/base_component.py similarity index 97% rename from sdk/python/kfp/components/base_component.py rename to sdk/python/kfp/dsl/base_component.py index 2b62ece34e..25a10f84df 100644 --- a/sdk/python/kfp/components/base_component.py +++ b/sdk/python/kfp/dsl/base_component.py @@ -16,9 +16,9 @@ import abc from typing import List -from kfp.components import pipeline_task -from kfp.components import structures -from kfp.components.types import type_utils +from kfp.dsl import pipeline_task +from kfp.dsl import structures +from kfp.dsl.types import type_utils from kfp.pipeline_spec import pipeline_spec_pb2 diff --git a/sdk/python/kfp/components/base_component_test.py b/sdk/python/kfp/dsl/base_component_test.py similarity index 95% rename from sdk/python/kfp/components/base_component_test.py rename to sdk/python/kfp/dsl/base_component_test.py index c9e0277564..48ed1d6065 100644 --- a/sdk/python/kfp/components/base_component_test.py +++ b/sdk/python/kfp/dsl/base_component_test.py @@ -11,18 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for kfp.components.base_component.""" +"""Tests for kfp.dsl.base_component.""" import unittest from unittest.mock import patch from kfp import dsl -from kfp.components import pipeline_task -from kfp.components import placeholders -from kfp.components import PythonComponent -from kfp.components import structures +from kfp.dsl import pipeline_task +from kfp.dsl import placeholders +from kfp.dsl import python_component +from kfp.dsl import structures -component_op = PythonComponent( +component_op = python_component.PythonComponent( # dummy python_func not used in behavior that is being tested python_func=lambda: None, component_spec=structures.ComponentSpec( diff --git a/sdk/python/kfp/components/component_decorator.py b/sdk/python/kfp/dsl/component_decorator.py similarity index 99% rename from sdk/python/kfp/components/component_decorator.py rename to sdk/python/kfp/dsl/component_decorator.py index 8bc64f17bc..7c6589589d 100644 --- a/sdk/python/kfp/components/component_decorator.py +++ b/sdk/python/kfp/dsl/component_decorator.py @@ -16,7 +16,7 @@ from typing import Callable, List, Optional import warnings -from kfp.components import component_factory +from kfp.dsl import component_factory def component(func: Optional[Callable] = None, diff --git a/sdk/python/kfp/components/component_decorator_test.py b/sdk/python/kfp/dsl/component_decorator_test.py similarity index 97% rename from sdk/python/kfp/components/component_decorator_test.py rename to sdk/python/kfp/dsl/component_decorator_test.py index e55d79af56..4b51de638f 100644 --- a/sdk/python/kfp/components/component_decorator_test.py +++ b/sdk/python/kfp/dsl/component_decorator_test.py @@ -17,9 +17,9 @@ from typing import Dict, List, NamedTuple import unittest -from kfp.components import python_component -from kfp.components import structures -from kfp.components.component_decorator import component +from kfp.dsl import python_component +from kfp.dsl import structures +from kfp.dsl.component_decorator import component class TestComponentDecorator(unittest.TestCase): diff --git a/sdk/python/kfp/components/component_factory.py b/sdk/python/kfp/dsl/component_factory.py similarity index 54% rename from sdk/python/kfp/components/component_factory.py rename to sdk/python/kfp/dsl/component_factory.py index 6585d0a813..5df3824e33 100644 --- a/sdk/python/kfp/components/component_factory.py +++ b/sdk/python/kfp/dsl/component_factory.py @@ -17,23 +17,26 @@ import pathlib import re import textwrap -from typing import Callable, List, Mapping, Optional, Tuple, Type, Union +from typing import (Any, Callable, Dict, List, Mapping, Optional, Tuple, Type, + Union) import warnings import docstring_parser -from kfp.components import container_component -from kfp.components import container_component_artifact_channel -from kfp.components import graph_component -from kfp.components import placeholders -from kfp.components import python_component -from kfp.components import structures -from kfp.components import task_final_status -from kfp.components.types import artifact_types -from kfp.components.types import custom_artifact_types -from kfp.components.types import type_annotations -from kfp.components.types import type_utils +import kfp +from kfp.dsl import container_component_artifact_channel +from kfp.dsl import container_component_class +from kfp.dsl import graph_component +from kfp.dsl import placeholders +from kfp.dsl import python_component +from kfp.dsl import structures +from kfp.dsl import task_final_status +from kfp.dsl.types import artifact_types +from kfp.dsl.types import custom_artifact_types +from kfp.dsl.types import type_annotations +from kfp.dsl.types import type_utils _DEFAULT_BASE_IMAGE = 'python:3.7' +SINGLE_OUTPUT_NAME = 'Output' @dataclasses.dataclass @@ -98,35 +101,74 @@ def make_index_url_options(pip_index_urls: Optional[List[str]]) -> str: return ' '.join(options) + ' ' +def make_pip_install_command( + install_parts: List[str], + index_url_options: str, +) -> str: + concat_package_list = ' '.join( + [repr(str(package)) for package in install_parts]) + return f'python3 -m pip install --quiet --no-warn-script-location {index_url_options}{concat_package_list}' + + _install_python_packages_script_template = ''' if ! [ -x "$(command -v pip)" ]; then python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip fi -PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \ - --no-warn-script-location {index_url_options}{concat_package_list} && "$0" "$@" +PIP_DISABLE_PIP_VERSION_CHECK=1 {pip_install_commands} && "$0" "$@" ''' def _get_packages_to_install_command( - package_list: Optional[List[str]] = None, - pip_index_urls: Optional[List[str]] = None) -> List[str]: - - if not package_list: + kfp_package_path: Optional[str] = None, + pip_index_urls: Optional[List[str]] = None, + packages_to_install: Optional[List[str]] = None, + install_kfp_package: bool = True, + target_image: Optional[str] = None, +) -> List[str]: + packages_to_install = packages_to_install or [] + kfp_in_user_pkgs = any(pkg.startswith('kfp') for pkg in packages_to_install) + # if the user doesn't say "don't install", they aren't building a + # container component, and they haven't already specified a KFP dep + # themselves, we install KFP for them + inject_kfp_install = install_kfp_package and target_image is None and not kfp_in_user_pkgs + if not inject_kfp_install and not packages_to_install: return [] - - concat_package_list = ' '.join( - [repr(str(package)) for package in package_list]) + pip_install_strings = [] index_url_options = make_index_url_options(pip_index_urls) - install_python_packages_script = _install_python_packages_script_template.format( - index_url_options=index_url_options, - concat_package_list=concat_package_list) - return ['sh', '-c', install_python_packages_script] + if inject_kfp_install: + if kfp_package_path: + kfp_pip_install_command = make_pip_install_command( + install_parts=[kfp_package_path], + index_url_options=index_url_options, + ) + else: + kfp_pip_install_command = make_pip_install_command( + install_parts=[ + f'kfp=={kfp.__version__}', + '--no-deps', + 'typing-extensions>=3.7.4,<5; python_version<"3.9"', + ], + index_url_options=index_url_options, + ) + pip_install_strings.append(kfp_pip_install_command) -def _get_default_kfp_package_path() -> str: - import kfp - return f'kfp=={kfp.__version__}' + if packages_to_install: + pip_install_strings.append(' && ') + + if packages_to_install: + user_packages_pip_install_command = make_pip_install_command( + install_parts=packages_to_install, + index_url_options=index_url_options, + ) + pip_install_strings.append(user_packages_pip_install_command) + + return [ + 'sh', '-c', + _install_python_packages_script_template.format( + pip_install_commands=' '.join(pip_install_strings)) + ] def _get_function_source_definition(func: Callable) -> str: @@ -151,7 +193,7 @@ def _get_function_source_definition(func: Callable) -> str: return '\n'.join(func_code_lines) -def _maybe_make_unique(name: str, names: List[str]): +def maybe_make_unique(name: str, names: List[str]): if name not in names: return name @@ -163,191 +205,187 @@ def _maybe_make_unique(name: str, names: List[str]): raise RuntimeError(f'Too many arguments with the name {name}') -def extract_component_interface( - func: Callable, +def get_name_to_specs( + signature: inspect.Signature, containerized: bool = False, - description: Optional[str] = None, - name: Optional[str] = None, -) -> structures.ComponentSpec: - single_output_name_const = 'Output' - - signature = inspect.signature(func) - parameters = list(signature.parameters.values()) - - original_docstring = inspect.getdoc(func) - parsed_docstring = docstring_parser.parse(original_docstring) - - inputs = {} - outputs = {} +) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """Returns two dictionaries. - input_names = set() - output_names = set() - for parameter in parameters: - parameter_type = type_annotations.maybe_strip_optional_from_annotation( - parameter.annotation) - passing_style = None - io_name = parameter.name - is_artifact_list = False - - if type_annotations.is_Input_Output_artifact_annotation(parameter_type): - # passing_style is either type_annotations.InputAnnotation or - # type_annotations.OutputAnnotation. - passing_style = type_annotations.get_io_artifact_annotation( - parameter_type) - - # parameter_type is a type like typing_extensions.Annotated[kfp.components.types.artifact_types.Artifact, ] OR typing_extensions.Annotated[typing.List[kfp.components.types.artifact_types.Artifact], ] - - is_artifact_list = type_annotations.is_list_of_artifacts( - parameter_type.__origin__) - - parameter_type = type_annotations.get_io_artifact_class( - parameter_type) - if not type_annotations.is_artifact_class(parameter_type): - raise ValueError( - f'Input[T] and Output[T] are only supported when T is an artifact or list of artifacts. Found `{io_name} with type {parameter_type}`' + The first is a mapping of input name to input annotation. The second + is a mapping of output name to output annotation. + """ + func_params = list(signature.parameters.values()) + + name_to_input_specs = {} + name_to_output_specs = {} + + ### handle function parameter annotations ### + for func_param in func_params: + name = func_param.name + if name == SINGLE_OUTPUT_NAME: + raise ValueError( + f'"{SINGLE_OUTPUT_NAME}" is an invalid parameter name.') + # Stripping Optional from Optional[] is the only processing done + # on annotations in this flow. Other than that, we extract the raw + # annotation and process later. + annotation = type_annotations.maybe_strip_optional_from_annotation( + func_param.annotation) + + # no annotation + if annotation == inspect._empty: + raise TypeError(f'Missing type annotation for argument: {name}') + + # is Input[Artifact], Input[List[]], (e.g., str), or InputPath() + elif (type_annotations.is_artifact_wrapped_in_Input(annotation) or + isinstance( + annotation, + type_annotations.InputPath, + ) or type_utils.is_parameter_type(annotation)): + name_to_input_specs[maybe_make_unique( + name, list(name_to_input_specs))] = make_input_spec( + annotation, func_param) + # is Artifact annotation (e.g., Artifact, Dataset, etc.) + # or List[] + elif type_annotations.issubclass_of_artifact( + annotation) or type_annotations.is_list_of_artifacts( + annotation): + if containerized: + raise TypeError( + f"Container Components must wrap input and output artifact annotations with Input/Output type markers (Input[] or Output[]). Got function input '{name}' with annotation {annotation}." ) + name_to_input_specs[maybe_make_unique( + name, list(name_to_input_specs))] = make_input_spec( + annotation, func_param) + + # is Output[Artifact] or OutputPath() + elif type_annotations.is_artifact_wrapped_in_Output( + annotation) or isinstance(annotation, + type_annotations.OutputPath): + name_to_output_specs[maybe_make_unique( + name, + list(name_to_output_specs))] = make_output_spec(annotation) + + # parameter type + else: + type_string = type_utils._annotation_to_type_struct(annotation) + name_to_input_specs[maybe_make_unique( + name, list(name_to_input_specs))] = make_input_spec( + type_string, func_param) - if parameter.default is not inspect.Parameter.empty: - if passing_style in [ - type_annotations.OutputAnnotation, - type_annotations.OutputPath, - ]: - raise ValueError( - 'Default values for Output artifacts are not supported.' - ) - elif parameter.default is not None: - raise ValueError( - f'Optional Input artifacts may only have default value None. Got: {parameter.default}.' - ) - - elif isinstance( - parameter_type, - (type_annotations.InputPath, type_annotations.OutputPath)): - passing_style = type(parameter_type) - parameter_type = parameter_type.type - if parameter.default is not inspect.Parameter.empty and not ( - passing_style == type_annotations.InputPath and - parameter.default is None): - raise ValueError( - 'Path inputs only support default values of None. Default' - ' values for outputs are not supported.') - - type_struct = type_utils._annotation_to_type_struct(parameter_type) - if type_struct is None: - raise TypeError( - f'Missing type annotation for argument: {parameter.name}') + ### handle return annotations ### + return_ann = signature.return_annotation - if passing_style in [ - type_annotations.OutputAnnotation, type_annotations.OutputPath + # validate container component returns + if containerized: + if return_ann not in [ + inspect.Parameter.empty, + structures.ContainerSpec, ]: - if io_name == single_output_name_const: - raise ValueError( - f'"{single_output_name_const}" is an invalid parameter name.' - ) - io_name = _maybe_make_unique(io_name, output_names) - output_names.add(io_name) - if type_annotations.is_artifact_class(parameter_type): - schema_version = parameter_type.schema_version - output_spec = structures.OutputSpec( - type=type_utils.create_bundled_artifact_type( - type_struct, schema_version), - is_artifact_list=is_artifact_list) - else: - output_spec = structures.OutputSpec(type=type_struct) - outputs[io_name] = output_spec - else: - io_name = _maybe_make_unique(io_name, input_names) - input_names.add(io_name) - type_ = type_utils.create_bundled_artifact_type( - type_struct, parameter_type.schema_version - ) if type_annotations.is_artifact_class( - parameter_type) else type_struct - default = None if parameter.default == inspect.Parameter.empty or type_annotations.is_artifact_class( - parameter_type) else parameter.default - optional = parameter.default is not inspect.Parameter.empty or type_utils.is_task_final_status_type( - type_struct) - input_spec = structures.InputSpec( - type=type_, - default=default, - optional=optional, - is_artifact_list=is_artifact_list, + raise TypeError( + 'Return annotation should be either ContainerSpec or omitted for container components.' ) + # ignore omitted returns + elif return_ann is None or return_ann == inspect.Parameter.empty: + pass + # is NamedTuple + elif hasattr(return_ann, '_fields'): + # Getting field type annotations. + # __annotations__ does not exist in python 3.5 and earlier + # _field_types does not exist in python 3.9 and later + field_annotations = getattr(return_ann, '__annotations__', + None) or getattr(return_ann, '_field_types') + for name in return_ann._fields: + annotation = field_annotations[name] + if not type_annotations.is_list_of_artifacts( + annotation) and not type_annotations.is_artifact_class( + annotation): + annotation = type_utils._annotation_to_type_struct(annotation) + name_to_output_specs[maybe_make_unique( + name, + list(name_to_output_specs))] = make_output_spec(annotation) + # is deprecated dict returns style + elif isinstance(return_ann, dict): + warnings.warn( + 'The ability to specify multiple outputs using the dict syntax' + ' has been deprecated. It will be removed soon after release' + ' 0.1.32. Please use typing.NamedTuple to declare multiple' + ' outputs.', DeprecationWarning) + for output_name, output_type_annotation in return_ann.items(): + output_type = type_utils._annotation_to_type_struct( + output_type_annotation) + name_to_output_specs[maybe_make_unique( + output_name, list(name_to_output_specs))] = output_type + # is the simple single return case (can be `-> ` or `-> Artifact`) + # treated the same way, since processing is done in inner functions + else: + name_to_output_specs[maybe_make_unique( + SINGLE_OUTPUT_NAME, + list(name_to_output_specs))] = make_output_spec(return_ann) + return name_to_input_specs, name_to_output_specs + + +def canonicalize_annotation(annotation: Any): + """Does cleaning on annotations that are common between input and output + annotations.""" + if type_annotations.is_Input_Output_artifact_annotation(annotation): + annotation = type_annotations.strip_Input_or_Output_marker(annotation) + if isinstance(annotation, + (type_annotations.InputPath, type_annotations.OutputPath)): + annotation = annotation.type + return annotation + + +def make_input_output_spec_args(annotation: Any) -> Dict[str, Any]: + """Gets a dict of kwargs shared between InputSpec and OutputSpec.""" + is_artifact_list = type_annotations.is_list_of_artifacts(annotation) + if is_artifact_list: + annotation = type_annotations.get_inner_type(annotation) + + if type_annotations.issubclass_of_artifact(annotation): + typ = type_utils.create_bundled_artifact_type(annotation.schema_title, + annotation.schema_version) + else: + typ = type_utils._annotation_to_type_struct(annotation) + return {'type': typ, 'is_artifact_list': is_artifact_list} - inputs[io_name] = input_spec - #Analyzing the return type annotations. - return_ann = signature.return_annotation - if not containerized: - if hasattr(return_ann, '_fields'): #NamedTuple - # Getting field type annotations. - # __annotations__ does not exist in python 3.5 and earlier - # _field_types does not exist in python 3.9 and later - field_annotations = getattr(return_ann, '__annotations__', - None) or getattr( - return_ann, '_field_types', None) - for field_name in return_ann._fields: - output_name = _maybe_make_unique(field_name, output_names) - output_names.add(output_name) - type_var = field_annotations.get(field_name) - if type_annotations.is_list_of_artifacts(type_var): - artifact_cls = type_var.__args__[0] - output_spec = structures.OutputSpec( - type=type_utils.create_bundled_artifact_type( - artifact_cls.schema_title, - artifact_cls.schema_version), - is_artifact_list=True) - elif type_annotations.is_artifact_class(type_var): - output_spec = structures.OutputSpec( - type=type_utils.create_bundled_artifact_type( - type_var.schema_title, type_var.schema_version)) - else: - type_struct = type_utils._annotation_to_type_struct( - type_var) - output_spec = structures.OutputSpec(type=type_struct) - outputs[output_name] = output_spec - # Deprecated dict-based way of declaring multiple outputs. Was only used by - # the @component decorator - elif isinstance(return_ann, dict): - warnings.warn( - 'The ability to specify multiple outputs using the dict syntax' - ' has been deprecated. It will be removed soon after release' - ' 0.1.32. Please use typing.NamedTuple to declare multiple' - ' outputs.') - for output_name, output_type_annotation in return_ann.items(): - output_type_struct = type_utils._annotation_to_type_struct( - output_type_annotation) - output_spec = structures.OutputSpec(type=output_type_struct) - outputs[name] = output_spec - elif signature.return_annotation is not None and signature.return_annotation != inspect.Parameter.empty: - output_name = _maybe_make_unique(single_output_name_const, - output_names) - # Fixes exotic, but possible collision: - # `def func(output_path: OutputPath()) -> str: ...` - output_names.add(output_name) - return_ann = signature.return_annotation - if type_annotations.is_list_of_artifacts(return_ann): - artifact_cls = return_ann.__args__[0] - output_spec = structures.OutputSpec( - type=type_utils.create_bundled_artifact_type( - artifact_cls.schema_title, artifact_cls.schema_version), - is_artifact_list=True) - elif type_annotations.is_artifact_class(return_ann): - output_spec = structures.OutputSpec( - type=type_utils.create_bundled_artifact_type( - return_ann.schema_title, return_ann.schema_version), - is_artifact_list=False) - else: - type_struct = type_utils._annotation_to_type_struct(return_ann) - output_spec = structures.OutputSpec(type=type_struct) - - outputs[output_name] = output_spec - elif return_ann != inspect.Parameter.empty and return_ann != structures.ContainerSpec: - raise TypeError( - 'Return annotation should be either ContainerSpec or omitted for container components.' +def make_output_spec(annotation: Any) -> structures.OutputSpec: + annotation = canonicalize_annotation(annotation) + args = make_input_output_spec_args(annotation) + return structures.OutputSpec(**args) + + +def make_input_spec(annotation: Any, + inspect_param: inspect.Parameter) -> structures.InputSpec: + """Makes an InputSpec from a cleaned output annotation.""" + annotation = canonicalize_annotation(annotation) + input_output_spec_args = make_input_output_spec_args(annotation) + + if (type_annotations.issubclass_of_artifact(annotation) or + input_output_spec_args['is_artifact_list'] + ) and inspect_param.default not in {None, inspect._empty}: + raise ValueError( + f'Optional Input artifacts may only have default value None. Got: {inspect_param.default}.' ) - component_name = name or _python_function_name_to_component_name( - func.__name__) + default = None if inspect_param.default == inspect.Parameter.empty or type_annotations.issubclass_of_artifact( + annotation) else inspect_param.default + + optional = inspect_param.default is not inspect.Parameter.empty or type_utils.is_task_final_status_type( + getattr(inspect_param.annotation, '__name__', '')) + return structures.InputSpec( + **input_output_spec_args, + default=default, + optional=optional, + ) + + +def extract_component_interface( + func: Callable, + containerized: bool = False, + description: Optional[str] = None, + name: Optional[str] = None, +) -> structures.ComponentSpec: def assign_descriptions( inputs_or_outputs: Mapping[str, Union[structures.InputSpec, @@ -363,7 +401,7 @@ def assign_descriptions( def parse_docstring_with_return_as_args( docstring: Union[str, - None]) -> Union[docstring_parser.Docstring, None]: + None]) -> Optional[docstring_parser.Docstring]: """Modifies docstring so that a return section can be treated as an args section, then parses the docstring.""" if docstring is None: @@ -379,23 +417,32 @@ def parse_docstring_with_return_as_args( return None - assign_descriptions(inputs, parsed_docstring.params) + signature = inspect.signature(func) + name_to_input_spec, name_to_output_spec = get_name_to_specs( + signature, containerized) + original_docstring = inspect.getdoc(func) + parsed_docstring = docstring_parser.parse(original_docstring) + + assign_descriptions(name_to_input_spec, parsed_docstring.params) modified_parsed_docstring = parse_docstring_with_return_as_args( original_docstring) if modified_parsed_docstring is not None: - assign_descriptions(outputs, modified_parsed_docstring.params) + assign_descriptions(name_to_output_spec, + modified_parsed_docstring.params) description = get_pipeline_description( decorator_description=description, docstring=parsed_docstring, ) + component_name = name or _python_function_name_to_component_name( + func.__name__) return structures.ComponentSpec( name=component_name, description=description, - inputs=inputs or None, - outputs=outputs or None, + inputs=name_to_input_spec or None, + outputs=name_to_output_spec or None, implementation=structures.Implementation(), ) @@ -420,8 +467,9 @@ def _get_command_and_args_for_lightweight_component( '-ec', textwrap.dedent('''\ program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main \ + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main \ --component_module_path \ "$program_path/ephemeral_component.py" \ "$@" @@ -444,7 +492,7 @@ def _get_command_and_args_for_containerized_component( command = [ 'python3', '-m', - 'kfp.components.executor_main', + 'kfp.dsl.executor_main', ] args = [ @@ -471,20 +519,25 @@ def create_component_from_func( The decorator is defined under component_decorator.py. See the decorator for the canonical documentation for this function. """ - packages_to_install = packages_to_install or [] - - if install_kfp_package and target_image is None: - if kfp_package_path is None: - kfp_package_path = _get_default_kfp_package_path() - packages_to_install.append(kfp_package_path) packages_to_install_command = _get_packages_to_install_command( - package_list=packages_to_install, pip_index_urls=pip_index_urls) + install_kfp_package=install_kfp_package, + target_image=target_image, + kfp_package_path=kfp_package_path, + packages_to_install=packages_to_install, + pip_index_urls=pip_index_urls, + ) command = [] args = [] if base_image is None: base_image = _DEFAULT_BASE_IMAGE + warnings.warn( + ("Python 3.7 has reached end-of-life. The default base_image used by the @dsl.component decorator will switch from 'python:3.7' to 'python:3.8' on April 23, 2024. To ensure your existing components work with versions of the KFP SDK released after that date, you should provide an explicit base_image argument and ensure your component works as intended on Python 3.8." + ), + FutureWarning, + stacklevel=2, + ) component_image = base_image @@ -535,7 +588,7 @@ def make_input_for_parameterized_container_component_function( Type[artifact_types.Artifact]] ) -> Union[placeholders.Placeholder, container_component_artifact_channel .ContainerComponentArtifactChannel]: - if type_annotations.is_input_artifact(annotation): + if type_annotations.is_artifact_wrapped_in_Input(annotation): if type_annotations.is_list_of_artifacts(annotation.__origin__): return placeholders.InputListOfArtifactsPlaceholder(name) @@ -543,7 +596,7 @@ def make_input_for_parameterized_container_component_function( return container_component_artifact_channel.ContainerComponentArtifactChannel( io_type='input', var_name=name) - elif type_annotations.is_output_artifact(annotation): + elif type_annotations.is_artifact_wrapped_in_Output(annotation): if type_annotations.is_list_of_artifacts(annotation.__origin__): return placeholders.OutputListOfArtifactsPlaceholder(name) @@ -569,7 +622,7 @@ def make_input_for_parameterized_container_component_function( def create_container_component_from_func( - func: Callable) -> container_component.ContainerComponent: + func: Callable) -> container_component_class.ContainerComponent: """Implementation for the @container_component decorator. The decorator is defined under container_component_decorator.py. See @@ -593,7 +646,7 @@ def create_container_component_from_func( component_spec.implementation = structures.Implementation( container_spec_implementation) component_spec._validate_placeholders() - return container_component.ContainerComponent(component_spec, func) + return container_component_class.ContainerComponent(component_spec, func) def create_graph_component_from_func( @@ -623,7 +676,7 @@ def create_graph_component_from_func( def get_pipeline_description( decorator_description: Union[str, None], docstring: docstring_parser.Docstring, -) -> Union[str, None]: +) -> Optional[str]: """Obtains the correct pipeline description from the pipeline decorator's description argument and the parsed docstring. diff --git a/sdk/python/kfp/dsl/component_factory_test.py b/sdk/python/kfp/dsl/component_factory_test.py new file mode 100644 index 0000000000..0def6344d6 --- /dev/null +++ b/sdk/python/kfp/dsl/component_factory_test.py @@ -0,0 +1,305 @@ +# Copyright 2022 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import List +import unittest + +from kfp import dsl +from kfp.dsl import component_factory +from kfp.dsl import Input +from kfp.dsl import Output +from kfp.dsl import structures +from kfp.dsl.component_decorator import component +from kfp.dsl.types.artifact_types import Artifact +from kfp.dsl.types.artifact_types import Model +from kfp.dsl.types.type_annotations import OutputPath + + +def strip_kfp_version(command: List[str]) -> List[str]: + return [ + re.sub(r"'kfp==(\d+).(\d+).(\d+)(-[a-z]+.\d+)?'", 'kfp', c) + for c in command + ] + + +class TestGetPackagesToInstallCommand(unittest.TestCase): + + def test_with_no_user_packages_to_install(self): + packages_to_install = [] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install) + self.assertEqual( + strip_kfp_version(command), + strip_kfp_version([ + 'sh', '-c', + '\nif ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \'kfp==2.1.2\' \'--no-deps\' \'typing-extensions>=3.7.4,<5; python_version<"3.9"\' && "$0" "$@"\n' + ])) + + def test_with_no_user_packages_to_install_and_install_kfp_false(self): + packages_to_install = [] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install, + install_kfp_package=False, + ) + self.assertEqual(command, []) + + def test_with_no_user_packages_to_install_and_kfp_package_path(self): + packages_to_install = [] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install, + kfp_package_path='git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python' + ) + + self.assertEqual( + strip_kfp_version(command), + strip_kfp_version([ + 'sh', '-c', + '\nif ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \'git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python\' && "$0" "$@"\n' + ])) + + def test_with_no_user_packages_to_install_and_kfp_package_path_and_install_kfp_false( + self): + packages_to_install = [] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install, + kfp_package_path='git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python', + install_kfp_package=False, + ) + self.assertEqual(command, []) + + def test_with_user_packages_to_install_and_kfp_package_path_and_install_kfp_false( + self): + packages_to_install = ['sklearn'] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install, + kfp_package_path='git+https://github.com/kubeflow/pipelines.git@master#subdirectory=sdk/python', + install_kfp_package=False, + ) + + self.assertEqual( + strip_kfp_version(command), + strip_kfp_version([ + 'sh', '-c', + '\nif ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \'sklearn\' && "$0" "$@"\n' + ])) + + def test_with_no_user_packages_to_install_and_kfp_package_path_and_target_image( + self): + packages_to_install = [] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install, + target_image='gcr.io/my-kfp-image', + kfp_package_path='./sdk/python') + + self.assertEqual(command, []) + + def test_with_no_user_packages_to_install_and_kfp_package_path_and_target_image_and_install_kfp_false( + self): + packages_to_install = [] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install, + target_image='gcr.io/my-kfp-image', + kfp_package_path='./sdk/python', + install_kfp_package=False) + + self.assertEqual(command, []) + + def test_with_user_packages_to_install_and_no_pip_index_url(self): + packages_to_install = ['package1', 'package2'] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install) + + self.assertEqual( + strip_kfp_version(command), + strip_kfp_version([ + 'sh', '-c', + '\nif ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \'kfp==2.1.3\' \'--no-deps\' \'typing-extensions>=3.7.4,<5; python_version<"3.9"\' && python3 -m pip install --quiet --no-warn-script-location \'package1\' \'package2\' && "$0" "$@"\n' + ])) + + def test_with_packages_to_install_with_pip_index_url(self): + packages_to_install = ['package1', 'package2'] + pip_index_urls = ['https://myurl.org/simple'] + + command = component_factory._get_packages_to_install_command( + packages_to_install=packages_to_install, + pip_index_urls=pip_index_urls, + ) + + self.assertEqual( + strip_kfp_version(command), + strip_kfp_version([ + 'sh', '-c', + '\nif ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://myurl.org/simple --trusted-host https://myurl.org/simple \'kfp==2.1.3\' \'--no-deps\' \'typing-extensions>=3.7.4,<5; python_version<"3.9"\' && python3 -m pip install --quiet --no-warn-script-location --index-url https://myurl.org/simple --trusted-host https://myurl.org/simple \'package1\' \'package2\' && "$0" "$@"\n' + ])) + + +class TestInvalidParameterName(unittest.TestCase): + + def test_output_named_Output(self): + + with self.assertRaisesRegex(ValueError, + r'"Output" is an invalid parameter name.'): + + @component + def comp(Output: OutputPath(str)): + pass + + def test_output_named_Output_with_string_output(self): + + with self.assertRaisesRegex(ValueError, + r'"Output" is an invalid parameter name.'): + + @component + def comp(Output: OutputPath(str), text: str) -> str: + pass + + +class TestExtractComponentInterfaceListofArtifacts(unittest.TestCase): + + def test_python_component_input(self): + + def comp(i: Input[List[Model]]): + ... + + component_spec = component_factory.extract_component_interface(comp) + self.assertEqual(component_spec.name, 'comp') + self.assertEqual(component_spec.description, None) + self.assertEqual( + component_spec.inputs, { + 'i': + structures.InputSpec( + type='system.Model@0.0.1', + default=None, + is_artifact_list=True) + }) + + def test_custom_container_component_input(self): + + def comp(i: Input[List[Artifact]]): + ... + + component_spec = component_factory.extract_component_interface( + comp, containerized=True) + self.assertEqual(component_spec.name, 'comp') + self.assertEqual(component_spec.description, None) + self.assertEqual( + component_spec.inputs, { + 'i': + structures.InputSpec( + type='system.Artifact@0.0.1', + default=None, + is_artifact_list=True) + }) + + def test_pipeline_input(self): + + def comp(i: Input[List[Model]]): + ... + + component_spec = component_factory.extract_component_interface(comp) + self.assertEqual(component_spec.name, 'comp') + self.assertEqual(component_spec.description, None) + self.assertEqual( + component_spec.inputs, { + 'i': + structures.InputSpec( + type='system.Model@0.0.1', + default=None, + is_artifact_list=True) + }) + + +class TestArtifactStringInInputpathOutputpath(unittest.TestCase): + + def test_unknown(self): + + @dsl.component + def comp( + i: dsl.InputPath('MyCustomType'), + o: dsl.OutputPath('MyCustomType'), + ): + ... + + self.assertEqual(comp.component_spec.outputs['o'].type, + 'system.Artifact@0.0.1') + self.assertFalse(comp.component_spec.outputs['o'].is_artifact_list) + self.assertEqual(comp.component_spec.inputs['i'].type, + 'system.Artifact@0.0.1') + self.assertFalse(comp.component_spec.inputs['i'].is_artifact_list) + + def test_known_v1_back_compat(self): + + @dsl.component + def comp( + i: dsl.InputPath('Dataset'), + o: dsl.OutputPath('Dataset'), + ): + ... + + self.assertEqual(comp.component_spec.outputs['o'].type, + 'system.Dataset@0.0.1') + self.assertFalse(comp.component_spec.outputs['o'].is_artifact_list) + self.assertEqual(comp.component_spec.inputs['i'].type, + 'system.Dataset@0.0.1') + self.assertFalse(comp.component_spec.inputs['i'].is_artifact_list) + + +class TestOutputListsOfArtifactsTemporarilyBlocked(unittest.TestCase): + + def test_python_component(self): + with self.assertRaisesRegex( + ValueError, + r"Output lists of artifacts are only supported for pipelines\. Got output list of artifacts for output parameter 'output_list' of component 'comp'\." + ): + + @dsl.component + def comp(output_list: Output[List[Artifact]]): + ... + + def test_container_component(self): + with self.assertRaisesRegex( + ValueError, + r"Output lists of artifacts are only supported for pipelines\. Got output list of artifacts for output parameter 'output_list' of component 'comp'\." + ): + + @dsl.container_component + def comp(output_list: Output[List[Artifact]]): + return dsl.ContainerSpec(image='alpine') + + +class TestPythonEOLWarning(unittest.TestCase): + + def test_default_base_image(self): + + with self.assertWarnsRegex( + FutureWarning, + r"Python 3\.7 has reached end-of-life\. The default base_image used by the @dsl\.component decorator will switch from 'python:3\.7' to 'python:3\.8' on April 23, 2024\. To ensure your existing components work with versions of the KFP SDK released after that date, you should provide an explicit base_image argument and ensure your component works as intended on Python 3\.8\." + ): + + @dsl.component + def foo(): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/sdk/python/kfp/components/constants.py b/sdk/python/kfp/dsl/constants.py similarity index 100% rename from sdk/python/kfp/components/constants.py rename to sdk/python/kfp/dsl/constants.py diff --git a/sdk/python/kfp/components/container_component_artifact_channel.py b/sdk/python/kfp/dsl/container_component_artifact_channel.py similarity index 80% rename from sdk/python/kfp/components/container_component_artifact_channel.py rename to sdk/python/kfp/dsl/container_component_artifact_channel.py index 84e0f984ec..322752295f 100644 --- a/sdk/python/kfp/components/container_component_artifact_channel.py +++ b/sdk/python/kfp/dsl/container_component_artifact_channel.py @@ -14,8 +14,6 @@ from typing import Union -from kfp.components import placeholders - class ContainerComponentArtifactChannel: """A class for passing in placeholders into container_component decorated @@ -25,13 +23,10 @@ def __init__(self, io_type: str, var_name: str): self._io_type = io_type self._var_name = var_name - def __getattr__( - self, _name: str - ) -> Union[placeholders.InputUriPlaceholder, placeholders - .InputPathPlaceholder, placeholders.OutputUriPlaceholder, - placeholders.OutputPathPlaceholder, - placeholders.InputMetadataPlaceholder, - placeholders.OutputMetadataPlaceholder]: + def __getattr__(self, _name: str) -> Union['placeholders.Placeholder']: + # aviod circular imports + from kfp.dsl import placeholders + attr_to_placeholder_dict = { 'uri': { 'input': placeholders.InputUriPlaceholder, diff --git a/sdk/python/kfp/components/container_component_artifact_channel_test.py b/sdk/python/kfp/dsl/container_component_artifact_channel_test.py similarity index 94% rename from sdk/python/kfp/components/container_component_artifact_channel_test.py rename to sdk/python/kfp/dsl/container_component_artifact_channel_test.py index 35eaeafdd6..fd04a027f0 100644 --- a/sdk/python/kfp/components/container_component_artifact_channel_test.py +++ b/sdk/python/kfp/dsl/container_component_artifact_channel_test.py @@ -14,8 +14,8 @@ import unittest -from kfp.components import container_component_artifact_channel -from kfp.components import placeholders +from kfp.dsl import container_component_artifact_channel +from kfp.dsl import placeholders class TestContainerComponentArtifactChannel(unittest.TestCase): diff --git a/sdk/python/kfp/components/container_component.py b/sdk/python/kfp/dsl/container_component_class.py similarity index 94% rename from sdk/python/kfp/components/container_component.py rename to sdk/python/kfp/dsl/container_component_class.py index 6c83ac3d06..7cd928036a 100644 --- a/sdk/python/kfp/components/container_component.py +++ b/sdk/python/kfp/dsl/container_component_class.py @@ -15,8 +15,8 @@ from typing import Callable -from kfp.components import base_component -from kfp.components import structures +from kfp.dsl import base_component +from kfp.dsl import structures class ContainerComponent(base_component.BaseComponent): diff --git a/sdk/python/kfp/components/container_component_decorator.py b/sdk/python/kfp/dsl/container_component_decorator.py similarity index 91% rename from sdk/python/kfp/components/container_component_decorator.py rename to sdk/python/kfp/dsl/container_component_decorator.py index 7d8901c97c..6ce43094ff 100644 --- a/sdk/python/kfp/components/container_component_decorator.py +++ b/sdk/python/kfp/dsl/container_component_decorator.py @@ -14,12 +14,12 @@ from typing import Callable -from kfp.components import component_factory -from kfp.components import container_component +from kfp.dsl import component_factory +from kfp.dsl import container_component_class def container_component( - func: Callable) -> container_component.ContainerComponent: + func: Callable) -> container_component_class.ContainerComponent: """Decorator for container-based components in KFP v2. Args: diff --git a/sdk/python/kfp/components/container_component_decorator_test.py b/sdk/python/kfp/dsl/container_component_decorator_test.py similarity index 94% rename from sdk/python/kfp/components/container_component_decorator_test.py rename to sdk/python/kfp/dsl/container_component_decorator_test.py index a234b16d22..d49253b1e8 100644 --- a/sdk/python/kfp/components/container_component_decorator_test.py +++ b/sdk/python/kfp/dsl/container_component_decorator_test.py @@ -16,8 +16,8 @@ import unittest from kfp import dsl -from kfp.components import container_component from kfp.dsl import Artifact +from kfp.dsl import container_component_class from kfp.dsl import Input from kfp.dsl import Output @@ -36,7 +36,7 @@ def hello_world() -> dsl.ContainerSpec: ) self.assertIsInstance(hello_world, - container_component.ContainerComponent) + container_component_class.ContainerComponent) self.assertIsNone(hello_world.component_spec.inputs) def test_func_with_simple_io(self): @@ -52,7 +52,7 @@ def hello_world_io( args=['--text', text, '--output_path', text_output_path]) self.assertIsInstance(hello_world_io, - container_component.ContainerComponent) + container_component_class.ContainerComponent) def test_func_with_artifact_io(self): @@ -78,7 +78,7 @@ def container_comp_with_artifacts( ]) self.assertIsInstance(container_comp_with_artifacts, - container_component.ContainerComponent) + container_component_class.ContainerComponent) class TestInputValuePlaceholderIrTypeHack(unittest.TestCase): diff --git a/sdk/python/kfp/dsl/executor.py b/sdk/python/kfp/dsl/executor.py new file mode 100644 index 0000000000..87d20e43c8 --- /dev/null +++ b/sdk/python/kfp/dsl/executor.py @@ -0,0 +1,431 @@ +# Copyright 2021 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import json +import os +import re +from typing import Any, Callable, Dict, List, Optional, Union +import warnings + +from kfp import dsl +from kfp.dsl import task_final_status +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_annotations + + +class Executor: + """Executor executes Python function components.""" + + def __init__( + self, + executor_input: Dict, + function_to_execute: Union[Callable, + 'python_component.PythonComponent'], + ): + + if hasattr(function_to_execute, 'python_func'): + self.func = function_to_execute.python_func + else: + self.func = function_to_execute + + self.executor_input = executor_input + self.executor_output_path = self.executor_input['outputs']['outputFile'] + + # drop executor_output.json part from the outputFile path + artifact_types.CONTAINER_TASK_ROOT = os.path.split( + self.executor_output_path)[0] + + self.input_artifacts: Dict[str, Union[dsl.Artifact, + List[dsl.Artifact]]] = {} + self.output_artifacts: Dict[str, dsl.Artifact] = {} + self.assign_input_and_output_artifacts() + + self.return_annotation = inspect.signature(self.func).return_annotation + self.excutor_output = {} + + def assign_input_and_output_artifacts(self) -> None: + for name, artifacts in self.executor_input.get('inputs', + {}).get('artifacts', + {}).items(): + list_of_artifact_proto_structs = artifacts.get('artifacts') + if list_of_artifact_proto_structs: + annotation = self.func.__annotations__[name] + # InputPath has no attribute __origin__ and also should be handled as a single artifact + annotation = type_annotations.maybe_strip_optional_from_annotation( + annotation) + is_list_of_artifacts = ( + type_annotations.is_Input_Output_artifact_annotation( + annotation) and + type_annotations.is_list_of_artifacts(annotation.__origin__) + ) or type_annotations.is_list_of_artifacts(annotation) + if is_list_of_artifacts: + self.input_artifacts[name] = [ + self.make_artifact( + msg, + name, + self.func, + ) for msg in list_of_artifact_proto_structs + ] + else: + self.input_artifacts[name] = self.make_artifact( + list_of_artifact_proto_structs[0], + name, + self.func, + ) + + for name, artifacts in self.executor_input.get('outputs', + {}).get('artifacts', + {}).items(): + list_of_artifact_proto_structs = artifacts.get('artifacts') + if list_of_artifact_proto_structs: + output_artifact = self.make_artifact( + list_of_artifact_proto_structs[0], + name, + self.func, + ) + self.output_artifacts[name] = output_artifact + makedirs_recursively(output_artifact.path) + + def make_artifact( + self, + runtime_artifact: Dict, + name: str, + func: Callable, + ) -> Any: + annotation = func.__annotations__.get(name) + if isinstance(annotation, type_annotations.InputPath): + schema_title, _ = annotation.type.split('@') + if schema_title in artifact_types._SCHEMA_TITLE_TO_TYPE: + artifact_cls = artifact_types._SCHEMA_TITLE_TO_TYPE[ + schema_title] + else: + raise TypeError( + f'Invalid type argument to {type_annotations.InputPath.__name__}: {annotation.type}' + ) + else: + artifact_cls = annotation + return create_artifact_instance( + runtime_artifact, artifact_cls=artifact_cls) + + def get_input_artifact(self, name: str) -> Optional[dsl.Artifact]: + return self.input_artifacts.get(name) + + def get_output_artifact(self, name: str) -> Optional[dsl.Artifact]: + return self.output_artifacts.get(name) + + def get_input_parameter_value(self, parameter_name: str) -> Optional[str]: + parameter_values = self.executor_input.get('inputs', {}).get( + 'parameterValues', None) + + if parameter_values is not None: + return parameter_values.get(parameter_name, None) + + return None + + def get_output_parameter_path(self, parameter_name: str) -> Optional[str]: + parameter = self.executor_input.get('outputs', {}).get( + 'parameters', {}).get(parameter_name, None) + if parameter is None: + return None + + path = parameter.get('outputFile', None) + if path: + makedirs_recursively(path) + return path + + def get_output_artifact_path(self, artifact_name: str) -> str: + output_artifact = self.output_artifacts.get(artifact_name) + if not output_artifact: + raise ValueError( + f'Failed to get output artifact path for artifact name {artifact_name}' + ) + return output_artifact.path + + def get_input_artifact_path(self, artifact_name: str) -> str: + input_artifact = self.input_artifacts.get(artifact_name) + if not input_artifact: + raise ValueError( + f'Failed to get input artifact path for artifact name {artifact_name}' + ) + return input_artifact.path + + def write_output_parameter_value( + self, name: str, value: Union[str, int, float, bool, dict, list, + Dict, List]) -> None: + if isinstance(value, (float, int)): + output = str(value) + elif isinstance(value, str): + # value is already a string. + output = value + elif isinstance(value, (bool, list, dict)): + output = json.dumps(value) + else: + raise ValueError( + f'Unable to serialize unknown type `{value}` for parameter input with value `{type(value)}`' + ) + + if not self.excutor_output.get('parameterValues'): + self.excutor_output['parameterValues'] = {} + + self.excutor_output['parameterValues'][name] = value + + def write_output_artifact_payload(self, name: str, value: Any) -> None: + path = self.get_output_artifact_path(name) + with open(path, 'w') as f: + f.write(str(value)) + + def handle_single_return_value(self, output_name: str, annotation_type: Any, + return_value: Any) -> None: + if is_parameter(annotation_type): + origin_type = getattr(annotation_type, '__origin__', + None) or annotation_type + # relax float-typed return to allow both int and float. + if origin_type == float: + accepted_types = (int, float) + # TODO: relax str-typed return to allow all primitive types? + else: + accepted_types = origin_type + if not isinstance(return_value, accepted_types): + raise ValueError( + f'Function `{self.func.__name__}` returned value of type {type(return_value)}; want type {origin_type}' + ) + self.write_output_parameter_value(output_name, return_value) + + elif is_artifact(annotation_type): + if isinstance(return_value, artifact_types.Artifact): + # for -> Artifact annotations, where the user returns an artifact + artifact_name = self.executor_input['outputs']['artifacts'][ + output_name]['artifacts'][0]['name'] + # users should not override the name for Vertex Pipelines + # if empty string, replace + # else provide descriptive warning and prefer letting backend throw exception + running_on_vertex = 'VERTEX_AI_PIPELINES_RUN_LABELS' in os.environ + if running_on_vertex: + if return_value.name == '': + return_value.name = artifact_name + else: + # prefer letting the backend throw the runtime exception + warnings.warn( + f'If you are running your pipeline Vertex AI Pipelines, you should not provide a name for your artifact. It will be set to the Vertex artifact resource name {artifact_name} by default. Got value for name: {return_value.name}.', + RuntimeWarning, + stacklevel=2) + self.output_artifacts[output_name] = return_value + else: + # for -> Artifact annotations, where the user returns some data that the executor should serialize + self.write_output_artifact_payload(output_name, return_value) + else: + raise RuntimeError( + f'Unknown return type: {annotation_type}. Must be one of the supported data types: https://www.kubeflow.org/docs/components/pipelines/v2/data-types/' + ) + + def write_executor_output(self, + func_output: Optional[Any] = None + ) -> Optional[str]: + """Writes executor output containing the Python function output. The + executor output file will not be written if this code is executed from + a non-chief node in a mirrored execution strategy. + + Args: + func_output: The object returned by the function. + + Returns: + Optional[str]: Returns the location of the executor_output file as a string if the file is written. Else, None. + """ + + if func_output is not None: + if is_parameter(self.return_annotation) or is_artifact( + self.return_annotation): + # Note: single output is named `Output` in component.yaml. + self.handle_single_return_value('Output', + self.return_annotation, + func_output) + elif is_named_tuple(self.return_annotation): + if len(self.return_annotation._fields) != len(func_output): + raise RuntimeError( + f'Expected {len(self.return_annotation._fields)} return values from function `{self.func.__name__}`, got {len(func_output)}' + ) + for i in range(len(self.return_annotation._fields)): + field = self.return_annotation._fields[i] + field_type = self.return_annotation.__annotations__[field] + if type(func_output) == tuple: + field_value = func_output[i] + else: + field_value = getattr(func_output, field) + self.handle_single_return_value(field, field_type, + field_value) + else: + raise RuntimeError( + f'Unknown return type: {self.return_annotation}. Must be one of `str`, `int`, `float`, a subclass of `Artifact`, or a NamedTuple collection of these types.' + ) + + if self.output_artifacts: + self.excutor_output['artifacts'] = {} + + for name, artifact in self.output_artifacts.items(): + runtime_artifact = { + 'name': artifact.name, + 'uri': artifact.uri, + 'metadata': artifact.metadata, + } + artifacts_list = {'artifacts': [runtime_artifact]} + + self.excutor_output['artifacts'][name] = artifacts_list + + # This check is to ensure only one worker (in a mirrored, distributed training/compute strategy) attempts to write to the same executor output file at the same time using gcsfuse, which enforces immutability of files. + write_file = True + + CLUSTER_SPEC_ENV_VAR_NAME = 'CLUSTER_SPEC' + cluster_spec_string = os.environ.get(CLUSTER_SPEC_ENV_VAR_NAME) + if cluster_spec_string: + cluster_spec = json.loads(cluster_spec_string) + CHIEF_NODE_LABELS = {'workerpool0', 'chief', 'master'} + write_file = cluster_spec['task']['type'] in CHIEF_NODE_LABELS + + if write_file: + makedirs_recursively(self.executor_output_path) + with open(self.executor_output_path, 'w') as f: + f.write(json.dumps(self.excutor_output)) + return self.executor_output_path + return None + + def execute(self) -> Optional[str]: + """Executes the function and writes the executor output file. The + executor output file will not be written if this code is executed from + a non-chief node in a mirrored execution strategy. + + Returns: + Optional[str]: Returns the location of the executor_output file as a string if the file is written. Else, None. + """ + annotations = inspect.getfullargspec(self.func).annotations + + # Function arguments. + func_kwargs = {} + + for k, v in annotations.items(): + if k == 'return': + continue + + # Annotations for parameter types could be written as, for example, + # `Optional[str]`. In this case, we need to strip off the part + # `Optional[]` to get the actual parameter type. + v = type_annotations.maybe_strip_optional_from_annotation(v) + + if v == task_final_status.PipelineTaskFinalStatus: + value = self.get_input_parameter_value(k) + func_kwargs[k] = task_final_status.PipelineTaskFinalStatus( + state=value.get('state'), + pipeline_job_resource_name=value.get( + 'pipelineJobResourceName'), + pipeline_task_name=value.get('pipelineTaskName'), + error_code=value.get('error').get('code', None), + error_message=value.get('error').get('message', None), + ) + + elif type_annotations.is_list_of_artifacts(v): + func_kwargs[k] = self.get_input_artifact(k) + + elif is_parameter(v): + value = self.get_input_parameter_value(k) + if value is not None: + func_kwargs[k] = value + + elif type_annotations.is_Input_Output_artifact_annotation(v): + if type_annotations.is_artifact_wrapped_in_Input(v): + func_kwargs[k] = self.get_input_artifact(k) + if type_annotations.is_artifact_wrapped_in_Output(v): + func_kwargs[k] = self.get_output_artifact(k) + + elif is_artifact(v): + func_kwargs[k] = self.get_input_artifact(k) + + elif isinstance(v, type_annotations.OutputPath): + if is_parameter(v.type): + func_kwargs[k] = self.get_output_parameter_path(k) + else: + func_kwargs[k] = self.get_output_artifact_path(k) + + elif isinstance(v, type_annotations.InputPath): + func_kwargs[k] = self.get_input_artifact_path(k) + + result = self.func(**func_kwargs) + return self.write_executor_output(result) + + +def create_artifact_instance( + runtime_artifact: Dict, + artifact_cls=dsl.Artifact, +) -> type: + """Creates an artifact class instances from a runtime artifact + dictionary.""" + schema_title = runtime_artifact.get('type', {}).get('schemaTitle', '') + + artifact_cls = artifact_types._SCHEMA_TITLE_TO_TYPE.get( + schema_title, artifact_cls) + return artifact_cls._from_executor_fields( + uri=runtime_artifact.get('uri', ''), + name=runtime_artifact.get('name', ''), + metadata=runtime_artifact.get('metadata', {}), + ) if hasattr(artifact_cls, '_from_executor_fields') else artifact_cls( + uri=runtime_artifact.get('uri', ''), + name=runtime_artifact.get('name', ''), + metadata=runtime_artifact.get('metadata', {}), + ) + + +def get_short_type_name(type_name: str) -> str: + """Extracts the short form type name. + + This method is used for looking up serializer for a given type. + + For example: + typing.List -> List + typing.List[int] -> List + typing.Dict[str, str] -> Dict + List -> List + str -> str + + Args: + type_name: The original type name. + + Returns: + The short form type name or the original name if pattern doesn't match. + """ + match = re.match('(typing\.)?(?P\w+)(?:\[.+\])?', type_name) + return match['type'] if match else type_name + + +# TODO: merge with type_utils.is_parameter_type +def is_parameter(annotation: Any) -> bool: + if type(annotation) == type: + return annotation in [str, int, float, bool, dict, list] + + # Annotation could be, for instance `typing.Dict[str, str]`, etc. + return get_short_type_name(str(annotation)) in ['Dict', 'List'] + + +def is_artifact(annotation: Any) -> bool: + if type(annotation) == type: + return type_annotations.is_artifact_class(annotation) + return False + + +def is_named_tuple(annotation: Any) -> bool: + if type(annotation) == type: + return issubclass(annotation, tuple) and hasattr( + annotation, '_fields') and hasattr(annotation, '__annotations__') + return False + + +def makedirs_recursively(path: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) diff --git a/sdk/python/kfp/components/executor_main.py b/sdk/python/kfp/dsl/executor_main.py similarity index 92% rename from sdk/python/kfp/components/executor_main.py rename to sdk/python/kfp/dsl/executor_main.py index 96943b0ca4..61e9406dd5 100644 --- a/sdk/python/kfp/components/executor_main.py +++ b/sdk/python/kfp/dsl/executor_main.py @@ -17,9 +17,9 @@ import os import sys -from kfp.components import executor as component_executor -from kfp.components import kfp_config -from kfp.components import utils +from kfp.dsl import executor as component_executor +from kfp.dsl import kfp_config +from kfp.dsl import utils def _setup_logging(): @@ -98,7 +98,11 @@ def executor_main(): executor = component_executor.Executor( executor_input=executor_input, function_to_execute=function_to_execute) - executor.execute() + output_file = executor.execute() + if output_file is None: + logging.info('Did not write output file.') + else: + logging.info(f'Wrote executor output file to {output_file}.') if __name__ == '__main__': diff --git a/sdk/python/kfp/components/executor_test.py b/sdk/python/kfp/dsl/executor_test.py similarity index 75% rename from sdk/python/kfp/components/executor_test.py rename to sdk/python/kfp/dsl/executor_test.py index 87625cc3b0..8b799d2c5a 100644 --- a/sdk/python/kfp/components/executor_test.py +++ b/sdk/python/kfp/dsl/executor_test.py @@ -11,8 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for kfp.components.executor.""" +"""Tests for kfp.dsl.executor.""" +import contextlib import json import os import tempfile @@ -21,17 +22,18 @@ from unittest import mock from absl.testing import parameterized -from kfp.components import executor -from kfp.components.task_final_status import PipelineTaskFinalStatus -from kfp.components.types import artifact_types -from kfp.components.types.artifact_types import Artifact -from kfp.components.types.artifact_types import Dataset -from kfp.components.types.artifact_types import Metrics -from kfp.components.types.artifact_types import Model -from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import OutputPath +from kfp import dsl +from kfp.dsl import executor from kfp.dsl import Input from kfp.dsl import Output +from kfp.dsl.task_final_status import PipelineTaskFinalStatus +from kfp.dsl.types import artifact_types +from kfp.dsl.types.artifact_types import Artifact +from kfp.dsl.types.artifact_types import Dataset +from kfp.dsl.types.artifact_types import Metrics +from kfp.dsl.types.artifact_types import Model +from kfp.dsl.types.type_annotations import InputPath +from kfp.dsl.types.type_annotations import OutputPath class ExecutorTest(parameterized.TestCase): @@ -40,10 +42,21 @@ class ExecutorTest(parameterized.TestCase): def setUp(cls): cls.maxDiff = None cls._test_dir = tempfile.mkdtemp() + + cls.prev_gcs_prefix = artifact_types._GCS_LOCAL_MOUNT_PREFIX + cls.prev_minio_prefix = artifact_types._MINIO_LOCAL_MOUNT_PREFIX + cls.prev_s3_prefix = artifact_types._S3_LOCAL_MOUNT_PREFIX + artifact_types._GCS_LOCAL_MOUNT_PREFIX = cls._test_dir + '/' artifact_types._MINIO_LOCAL_MOUNT_PREFIX = cls._test_dir + '/minio/' artifact_types._S3_LOCAL_MOUNT_PREFIX = cls._test_dir + '/s3/' + @classmethod + def tearDown(cls): + artifact_types._GCS_LOCAL_MOUNT_PREFIX = cls.prev_gcs_prefix + artifact_types._MINIO_LOCAL_MOUNT_PREFIX = cls.prev_minio_prefix + artifact_types._S3_LOCAL_MOUNT_PREFIX = cls.prev_s3_prefix + def execute(self, func: Callable, executor_input: str) -> None: executor_input_dict = json.loads(executor_input % {'test_dir': self._test_dir}) @@ -51,12 +64,12 @@ def execute(self, func: Callable, executor_input: str) -> None: executor.Executor( executor_input=executor_input_dict, function_to_execute=func).execute() + return executor_input_dict['outputs']['outputFile'] def execute_and_load_output_metadata(self, func: Callable, executor_input: str) -> dict: - self.execute(func, executor_input) - with open(os.path.join(self._test_dir, 'output_metadata.json'), - 'r') as f: + output_file = self.execute(func, executor_input) + with open(output_file) as f: return json.loads(f.read()) def test_input_and_output_parameters(self): @@ -113,15 +126,10 @@ def test_input_artifact_custom_type(self): } """ - class VertexDataset: + class VertexDataset(dsl.Artifact): schema_title = 'google.VertexDataset' schema_version = '0.0.0' - def __init__(self, name: str, uri: str, metadata: dict) -> None: - self.name = name - self.uri = uri - self.metadata = metadata - @property def path(self) -> str: return self.uri.replace('gs://', @@ -1157,6 +1165,39 @@ def test_func(input_artifact: Input[Artifact]): input_artifact.name, 'projects/123/locations/us-central1/metadataStores/default/artifacts/input_artifact' ) + + output_metadata = self.execute_and_load_output_metadata( + test_func, executor_input) + + self.assertDictEqual(output_metadata, {}) + + def test_single_artifact_input_pythonic(self): + executor_input = """\ + { + "inputs": { + "artifacts": { + "input_artifact": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/input_artifact", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output/input_artifact" + } + ] + } + } + }, + "outputs": { + "outputFile": "%(test_dir)s/output_metadata.json" + } + } + """ + + def test_func(input_artifact: Artifact): + self.assertIsInstance(input_artifact, Artifact) self.assertEqual( input_artifact.name, 'projects/123/locations/us-central1/metadataStores/default/artifacts/input_artifact' @@ -1167,6 +1208,261 @@ def test_func(input_artifact: Input[Artifact]): self.assertDictEqual(output_metadata, {}) + def test_single_artifact_input_pythonic_with_optional(self): + executor_input = """\ + { + "inputs": { + "artifacts": { + "input_artifact": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/input_artifact", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output/input_artifact" + } + ] + } + } + }, + "outputs": { + "outputFile": "%(test_dir)s/output_metadata.json" + } + } + """ + + def test_func(input_artifact: Optional[Artifact] = None): + self.assertIsInstance(input_artifact, Artifact) + self.assertEqual( + input_artifact.name, + 'projects/123/locations/us-central1/metadataStores/default/artifacts/input_artifact' + ) + + output_metadata = self.execute_and_load_output_metadata( + test_func, executor_input) + + self.assertDictEqual(output_metadata, {}) + + def test_single_artifact_output_pythonic(self): + executor_input = """\ + { + "inputs": {}, + "outputs": { + "artifacts": { + "Output": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/123", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output" + } + ] + } + }, + "outputFile": "%(test_dir)s/output_metadata.json" + } + } + """ + + def test_func() -> Artifact: + return Artifact( + uri='gs://manually_specified_bucket/foo', + metadata={'data': 123}, + ) + + with temporary_envvar('VERTEX_AI_PIPELINES_RUN_LABELS', '12325'): + output_metadata = self.execute_and_load_output_metadata( + test_func, executor_input) + + self.assertDictEqual( + output_metadata, { + 'artifacts': { + 'Output': { + 'artifacts': [{ + 'name': + 'projects/123/locations/us-central1/metadataStores/default/artifacts/123', + 'uri': + 'gs://manually_specified_bucket/foo', + 'metadata': { + 'data': 123 + } + }] + } + }, + }) + + def test_single_artifact_output_pythonic_with_get_uri(self): + executor_input = """\ + { + "inputs": {}, + "outputs": { + "artifacts": { + "Output": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/123", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output" + } + ] + } + }, + "outputFile": "%(test_dir)s/another_bucket/output_metadata.json" + } + } + """ + + def test_func() -> Artifact: + return Artifact( + uri=dsl.get_uri(suffix='my_artifact'), + metadata={'data': 123}, + ) + + with temporary_envvar('VERTEX_AI_PIPELINES_RUN_LABELS', '12325'): + output_metadata = self.execute_and_load_output_metadata( + test_func, executor_input) + + self.assertDictEqual( + output_metadata, { + 'artifacts': { + 'Output': { + 'artifacts': [{ + 'name': + 'projects/123/locations/us-central1/metadataStores/default/artifacts/123', + 'uri': + 'gs://another_bucket/my_artifact', + 'metadata': { + 'data': 123 + } + }] + } + }, + }) + + def test_multiple_artifact_output_pythonic_with_get_uri(self): + executor_input = """\ + { + "inputs": {}, + "outputs": { + "artifacts": { + "a": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/123", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output" + } + ] + }, + "d": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/321", + "type": { + "schemaTitle": "system.Dataset" + }, + "uri": "gs://some-bucket/output" + } + ] + } + }, + "outputFile": "%(test_dir)s/another_bucket/output_metadata.json" + } + } + """ + + def test_func() -> NamedTuple('outputs', a=Artifact, d=Dataset): + outputs = NamedTuple('outputs', a=Artifact, d=Dataset) + return outputs( + a=Artifact( + uri=dsl.get_uri(suffix='artifact'), + metadata={'data': 123}, + ), + d=Dataset( + uri=dsl.get_uri(suffix='dataset'), + metadata={}, + )) + + with temporary_envvar('VERTEX_AI_PIPELINES_RUN_LABELS', '12325'): + output_metadata = self.execute_and_load_output_metadata( + test_func, executor_input) + + self.assertDictEqual( + output_metadata, { + 'artifacts': { + 'a': { + 'artifacts': [{ + 'name': + 'projects/123/locations/us-central1/metadataStores/default/artifacts/123', + 'uri': + 'gs://another_bucket/artifact', + 'metadata': { + 'data': 123 + } + }] + }, + 'd': { + 'artifacts': [{ + 'name': + 'projects/123/locations/us-central1/metadataStores/default/artifacts/321', + 'uri': + 'gs://another_bucket/dataset', + 'metadata': {} + }] + } + }, + }) + + def test_warns_if_artifact_name_for_vertex(self): + executor_input = """\ + { + "inputs": {}, + "outputs": { + "artifacts": { + "Output": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/123", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output" + } + ] + } + }, + "outputFile": "%(test_dir)s/output_metadata.json" + } + } + """ + + def test_func() -> Artifact: + return Artifact( + name='illegal_custom_name', + uri='gs://manually_specified_bucket/foo', + metadata={'data': 123}, + ) + + with temporary_envvar('VERTEX_AI_PIPELINES_RUN_LABELS', '12325'): + with self.assertWarnsRegex( + RuntimeWarning, + r'If you are running your pipeline Vertex AI Pipelines, you should not provide a name for your artifact\. It will be set to the Vertex artifact resource name projects/123/locations/us-central1/metadataStores/default/artifacts/123 by default\. Got value for name: illegal_custom_name\.' + ): + self.execute_and_load_output_metadata(test_func, executor_input) + def test_list_of_artifacts_input(self): executor_input = """\ { @@ -1216,28 +1512,103 @@ def test_func(input_list: Input[List[Artifact]]): self.assertDictEqual(output_metadata, {}) + def test_list_of_artifacts_input_pythonic(self): + executor_input = """\ + { + "inputs": { + "artifacts": { + "input_list": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/0", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output/input_list/0" + }, + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/1", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output/input_list/1" + } + ] + } + } + }, + "outputs": { + "outputFile": "%(test_dir)s/output_metadata.json" + } + } + """ + + def test_func(input_list: List[Artifact]): + self.assertEqual(len(input_list), 2) + self.assertEqual( + input_list[0].name, + 'projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/0' + ) + self.assertEqual( + input_list[1].name, + 'projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/1' + ) + + output_metadata = self.execute_and_load_output_metadata( + test_func, executor_input) -class VertexDataset: - schema_title = 'google.VertexDataset' - schema_version = '0.0.0' + self.assertDictEqual(output_metadata, {}) - @classmethod - def _from_executor_fields( - cls, - name: str, - uri: str, - metadata: dict, - ) -> 'VertexDataset': + def test_list_of_artifacts_input_pythonic_with_optional(self): + executor_input = """\ + { + "inputs": { + "artifacts": { + "input_list": { + "artifacts": [ + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/0", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output/input_list/0" + }, + { + "metadata": {}, + "name": "projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/1", + "type": { + "schemaTitle": "system.Artifact" + }, + "uri": "gs://some-bucket/output/input_list/1" + } + ] + } + } + }, + "outputs": { + "outputFile": "%(test_dir)s/output_metadata.json" + } + } + """ - instance = VertexDataset() - instance.name = name - instance.uri = uri - instance.metadata = metadata - return instance + def test_func(input_list: List[Artifact] = None): + self.assertEqual(len(input_list), 2) + self.assertEqual( + input_list[0].name, + 'projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/0' + ) + self.assertEqual( + input_list[1].name, + 'projects/123/locations/us-central1/metadataStores/default/artifacts/input_list/1' + ) + + output_metadata = self.execute_and_load_output_metadata( + test_func, executor_input) - @property - def path(self) -> str: - return self.uri.replace('gs://', artifact_types._GCS_LOCAL_MOUNT_PREFIX) + self.assertDictEqual(output_metadata, {}) class TestDictToArtifact(parameterized.TestCase): @@ -1355,24 +1726,23 @@ def test_dict_to_artifact_kfp_artifact( self.assertIsInstance( executor.create_artifact_instance(runtime_artifact), expected_type) - def test_dict_to_artifact_google_artifact(self): - runtime_artifact = { - 'metadata': {}, - 'name': 'input_artifact_one', - 'type': { - 'schemaTitle': 'google.VertexDataset' - }, - 'uri': 'gs://some-bucket/input_artifact_one' - } - # with artifact_cls - self.assertIsInstance( - executor.create_artifact_instance( - runtime_artifact, artifact_cls=VertexDataset), VertexDataset) - # without artifact_cls - self.assertIsInstance( - executor.create_artifact_instance(runtime_artifact), - artifact_types.Artifact) +@contextlib.contextmanager +def temporary_envvar(key: str, value: str) -> None: + # Save the old value if it exists + old_value = os.environ.get(key, None) + + # Set the new value + os.environ[key] = value + + try: + yield + finally: + # Restore the old value or delete the key if it didn't exist before + if old_value is not None: + os.environ[key] = old_value + else: + del os.environ[key] if __name__ == '__main__': diff --git a/sdk/python/kfp/components/for_loop.py b/sdk/python/kfp/dsl/for_loop.py similarity index 88% rename from sdk/python/kfp/components/for_loop.py rename to sdk/python/kfp/dsl/for_loop.py index f838dff257..14e0a42773 100644 --- a/sdk/python/kfp/components/for_loop.py +++ b/sdk/python/kfp/dsl/for_loop.py @@ -16,7 +16,7 @@ import re from typing import Any, Dict, List, Optional, Union -from kfp.components import pipeline_channel +from kfp.dsl import pipeline_channel ItemList = List[Union[int, float, str, Dict[str, Any]]] @@ -77,7 +77,7 @@ class LoopArgument(pipeline_channel.PipelineParameterChannel): Attributes: - items_or_pipeline_channel: The raw items or the PipelineChannel object + items_or_pipeline_channel: The raw items or the PipelineParameterChannel object this LoopArgument is associated to. """ LOOP_ITEM_NAME_BASE = 'loop-item' @@ -85,7 +85,7 @@ class LoopArgument(pipeline_channel.PipelineParameterChannel): def __init__( self, - items: Union[ItemList, pipeline_channel.PipelineChannel], + items: Union[ItemList, pipeline_channel.PipelineParameterChannel], name_code: Optional[str] = None, name_override: Optional[str] = None, **kwargs, @@ -99,8 +99,8 @@ def __init__( name_code: A unique code used to identify these loop arguments. Should match the code for the ParallelFor ops_group which created these LoopArguments. This prevents parameter name collisions. - name_override: The override name for PipelineChannel. - **kwargs: Any other keyword arguments passed down to PipelineChannel. + name_override: The override name for PipelineParameterChannel. + **kwargs: Any other keyword arguments passed down to PipelineParameterChannel. """ if (name_code is None) == (name_override is None): raise ValueError( @@ -112,17 +112,19 @@ def __init__( else: super().__init__(name=name_override, **kwargs) - if not isinstance(items, - (list, tuple, pipeline_channel.PipelineChannel)): + if not isinstance( + items, + (list, tuple, pipeline_channel.PipelineParameterChannel)): raise TypeError( - f'Expected list, tuple, or PipelineChannel, got {items}.') + f'Expected list, tuple, or PipelineParameterChannel, got {items}.' + ) if isinstance(items, tuple): items = list(items) self.items_or_pipeline_channel = items self.is_with_items_loop_argument = not isinstance( - items, pipeline_channel.PipelineChannel) + items, pipeline_channel.PipelineParameterChannel) self._referenced_subvars: Dict[str, LoopArgumentVariable] = {} if isinstance(items, list) and isinstance(items[0], dict): @@ -154,9 +156,10 @@ def _make_name(self, code: str): @classmethod def from_pipeline_channel( cls, - channel: pipeline_channel.PipelineChannel, + channel: pipeline_channel.PipelineParameterChannel, ) -> 'LoopArgument': - """Creates a LoopArgument object from a PipelineChannel object.""" + """Creates a LoopArgument object from a PipelineParameterChannel + object.""" return LoopArgument( items=channel, name_override=channel.name + '-' + cls.LOOP_ITEM_NAME_BASE, @@ -191,7 +194,7 @@ def name_is_loop_argument(cls, name: str) -> bool: or (cls.LOOP_ITEM_PARAM_NAME_BASE + '-') in name -class LoopArgumentVariable(pipeline_channel.PipelineChannel): +class LoopArgumentVariable(pipeline_channel.PipelineParameterChannel): """Represents a subvariable for a loop argument. This is used for cases where we're looping over maps, each of which contains @@ -246,7 +249,7 @@ def __init__( @property def items_or_pipeline_channel( - self) -> Union[ItemList, pipeline_channel.PipelineChannel]: + self) -> Union[ItemList, pipeline_channel.PipelineParameterChannel]: """Returns the loop argument items.""" return self.loop_argument.items_or_pipeline_chanenl @@ -274,6 +277,7 @@ def _get_name_override(self, loop_arg_name: str, subvar_name: str) -> str: return f'{loop_arg_name}{self.SUBVAR_NAME_DELIMITER}{subvar_name}' +# TODO: migrate Collected to OneOfMixin style implementation class Collected(pipeline_channel.PipelineChannel): """For collecting into a list the output from a task in dsl.ParallelFor loops. @@ -313,3 +317,13 @@ def __init__( channel_type=channel_type, task_name=output.task_name, ) + self._validate_no_oneof_channel(self.output) + + def _validate_no_oneof_channel( + self, channel: Union[pipeline_channel.PipelineParameterChannel, + pipeline_channel.PipelineArtifactChannel] + ) -> None: + if isinstance(channel, pipeline_channel.OneOfMixin): + raise ValueError( + f'dsl.{pipeline_channel.OneOf.__name__} cannot be used inside of dsl.{Collected.__name__}.' + ) diff --git a/sdk/python/kfp/components/for_loop_test.py b/sdk/python/kfp/dsl/for_loop_test.py similarity index 98% rename from sdk/python/kfp/components/for_loop_test.py rename to sdk/python/kfp/dsl/for_loop_test.py index b1e3549c94..7d1559c87b 100644 --- a/sdk/python/kfp/components/for_loop_test.py +++ b/sdk/python/kfp/dsl/for_loop_test.py @@ -15,8 +15,8 @@ import unittest from absl.testing import parameterized -from kfp.components import for_loop -from kfp.components import pipeline_channel +from kfp.dsl import for_loop +from kfp.dsl import pipeline_channel class ForLoopTest(parameterized.TestCase): diff --git a/sdk/python/kfp/components/graph_component.py b/sdk/python/kfp/dsl/graph_component.py similarity index 95% rename from sdk/python/kfp/components/graph_component.py rename to sdk/python/kfp/dsl/graph_component.py index 181ae809a8..2b09927dfa 100644 --- a/sdk/python/kfp/components/graph_component.py +++ b/sdk/python/kfp/dsl/graph_component.py @@ -18,10 +18,10 @@ import uuid from kfp.compiler import pipeline_spec_builder as builder -from kfp.components import base_component -from kfp.components import pipeline_channel -from kfp.components import pipeline_context -from kfp.components import structures +from kfp.dsl import base_component +from kfp.dsl import pipeline_channel +from kfp.dsl import pipeline_context +from kfp.dsl import structures from kfp.pipeline_spec import pipeline_spec_pb2 diff --git a/sdk/python/kfp/components/importer_component.py b/sdk/python/kfp/dsl/importer_component.py similarity index 92% rename from sdk/python/kfp/components/importer_component.py rename to sdk/python/kfp/dsl/importer_component.py index ac6e14614a..168c7c6f73 100644 --- a/sdk/python/kfp/components/importer_component.py +++ b/sdk/python/kfp/dsl/importer_component.py @@ -13,8 +13,8 @@ # limitations under the License. """Importer-based component.""" -from kfp.components import base_component -from kfp.components import structures +from kfp.dsl import base_component +from kfp.dsl import structures class ImporterComponent(base_component.BaseComponent): diff --git a/sdk/python/kfp/components/importer_node.py b/sdk/python/kfp/dsl/importer_node.py similarity index 93% rename from sdk/python/kfp/components/importer_node.py rename to sdk/python/kfp/dsl/importer_node.py index d62edf7917..2a3e676daa 100644 --- a/sdk/python/kfp/components/importer_node.py +++ b/sdk/python/kfp/dsl/importer_node.py @@ -15,14 +15,14 @@ from typing import Any, Dict, Mapping, Optional, Type, Union -from kfp.components import importer_component -from kfp.components import pipeline_channel -from kfp.components import pipeline_task -from kfp.components import placeholders -from kfp.components import structures -from kfp.components import utils -from kfp.components.types import artifact_types -from kfp.components.types import type_utils +from kfp.dsl import importer_component +from kfp.dsl import pipeline_channel +from kfp.dsl import pipeline_task +from kfp.dsl import placeholders +from kfp.dsl import structures +from kfp.dsl import utils +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_utils URI_KEY = 'uri' OUTPUT_KEY = 'artifact' diff --git a/sdk/python/kfp/components/importer_node_test.py b/sdk/python/kfp/dsl/importer_node_test.py similarity index 84% rename from sdk/python/kfp/components/importer_node_test.py rename to sdk/python/kfp/dsl/importer_node_test.py index d6066b5a4d..4b3493bfc8 100644 --- a/sdk/python/kfp/components/importer_node_test.py +++ b/sdk/python/kfp/dsl/importer_node_test.py @@ -14,8 +14,8 @@ import unittest from kfp import dsl -from kfp.components import importer_node -from kfp.components.types.artifact_types import Dataset +from kfp.dsl import Dataset +from kfp.dsl import importer_node class TestImporterSupportsDynamicMetadata(unittest.TestCase): @@ -184,3 +184,37 @@ def my_pipeline(integer: int = 1): "prefix2-{{$.inputs.parameters[\'metadata-2\']}}") self.assertEqual(metadata.struct_value.fields['key'].string_value, 'value') + + def test_uri_from_loop(self): + + @dsl.component + def make_args() -> list: + return [{'uri': 'gs://foo', 'key': 'foo'}] + + @dsl.pipeline + def my_pipeline(): + with dsl.ParallelFor(make_args().output) as data: + dsl.importer( + artifact_uri=data.uri, + artifact_class=Dataset, + metadata={'metadata_key': data.key}) + + self.assertEqual( + my_pipeline.pipeline_spec.deployment_spec['executors'] + ['exec-importer']['importer']['artifactUri']['runtimeParameter'], + 'uri') + self.assertEqual( + my_pipeline.pipeline_spec.deployment_spec['executors'] + ['exec-importer']['importer']['metadata']['metadata_key'], + "{{$.inputs.parameters[\'metadata\']}}") + self.assertEqual( + my_pipeline.pipeline_spec.components['comp-for-loop-1'].dag + .tasks['importer'].inputs.parameters['metadata'] + .component_input_parameter, + 'pipelinechannel--make-args-Output-loop-item') + self.assertEqual( + my_pipeline.pipeline_spec.components['comp-for-loop-1'].dag + .tasks['importer'].inputs.parameters['metadata'] + .parameter_expression_selector, + 'parseJson(string_value)["key"]', + ) diff --git a/sdk/python/kfp/components/kfp_config.py b/sdk/python/kfp/dsl/kfp_config.py similarity index 100% rename from sdk/python/kfp/components/kfp_config.py rename to sdk/python/kfp/dsl/kfp_config.py diff --git a/sdk/python/kfp/components/pipeline_channel.py b/sdk/python/kfp/dsl/pipeline_channel.py similarity index 52% rename from sdk/python/kfp/components/pipeline_channel.py rename to sdk/python/kfp/dsl/pipeline_channel.py index 26ad27eae3..6adb52525c 100644 --- a/sdk/python/kfp/components/pipeline_channel.py +++ b/sdk/python/kfp/dsl/pipeline_channel.py @@ -20,21 +20,24 @@ import re from typing import Dict, List, Optional, Union -from kfp.components.types import type_utils +from kfp.dsl.types import type_utils @dataclasses.dataclass -class ConditionOperator: - """Represents a condition expression to be used in dsl.Condition(). +class ConditionOperation: + """Represents a condition expression to be used in condition control flow + group. Attributes: operator: The operator of the condition. left_operand: The left operand. right_operand: The right operand. + negate: Whether to negate the result of the binary operation. """ operator: str left_operand: Union['PipelineParameterChannel', type_utils.PARAMETER_TYPES] right_operand: Union['PipelineParameterChannel', type_utils.PARAMETER_TYPES] + negate: bool = False # The string template used to generate the placeholder of a PipelineChannel. @@ -97,14 +100,33 @@ def __init__( # so that serialization and unserialization remain consistent # (i.e. None => '' => None) self.task_name = task_name or None - from kfp.components import pipeline_context + from kfp.dsl import pipeline_context - default_pipeline = pipeline_context.Pipeline.get_default_pipeline() - if self.task_name is not None and default_pipeline is not None and default_pipeline.tasks: - self.task = pipeline_context.Pipeline.get_default_pipeline().tasks[ - self.task_name] - else: - self.task = None + self.pipeline = pipeline_context.Pipeline.get_default_pipeline() + + @property + def task(self) -> Union['PipelineTask', 'TasksGroup']: + # TODO: migrate Collected to OneOfMixin style implementation, + # then move this out of a property + if self.task_name is None or self.pipeline is None: + return None + + if self.task_name in self.pipeline.tasks: + return self.pipeline.tasks[self.task_name] + + from kfp.compiler import compiler_utils + all_groups = compiler_utils.get_all_groups(self.pipeline.groups[0]) + # pipeline hasn't exited, so it doesn't have a name + all_groups_no_pipeline = all_groups[1:] + group_name_to_group = { + group.name: group for group in all_groups_no_pipeline + } + if self.task_name in group_name_to_group: + return group_name_to_group[self.task_name] + + raise ValueError( + f"PipelineChannel task name '{self.task_name}' not found in pipeline." + ) @property def full_name(self) -> str: @@ -149,22 +171,22 @@ def __hash__(self) -> int: return hash(self.pattern) def __eq__(self, other): - return ConditionOperator('==', self, other) + return ConditionOperation('==', self, other) def __ne__(self, other): - return ConditionOperator('!=', self, other) + return ConditionOperation('!=', self, other) def __lt__(self, other): - return ConditionOperator('<', self, other) + return ConditionOperation('<', self, other) def __le__(self, other): - return ConditionOperator('<=', self, other) + return ConditionOperation('<=', self, other) def __gt__(self, other): - return ConditionOperator('>', self, other) + return ConditionOperation('>', self, other) def __ge__(self, other): - return ConditionOperator('>=', self, other) + return ConditionOperation('>=', self, other) class PipelineParameterChannel(PipelineChannel): @@ -262,6 +284,228 @@ def __init__( ) +class OneOfMixin(PipelineChannel): + """Shared functionality for OneOfParameter and OneOfAritfact.""" + + def _set_condition_branches_group( + self, channels: List[Union[PipelineParameterChannel, + PipelineArtifactChannel]] + ) -> None: + # avoid circular import + from kfp.dsl import tasks_group + + # .condition_branches_group could really be collapsed into just .task, + # but we prefer keeping both for clarity in the rest of the compiler + # code. When the code is logically related to a + # condition_branches_group, it aids understanding to reference this + # attribute name. When the code is trying to treat the OneOfMixin like + # a typical PipelineChannel, it aids to reference task. + self.condition_branches_group: tasks_group.ConditionBranches = channels[ + 0].task.parent_task_group.parent_task_group + + def _make_oneof_name(self) -> str: + # avoid circular imports + from kfp.compiler import compiler_utils + + # This is a different type of "injected channel". + # We know that this output will _always_ be a pipeline channel, so we + # set the pipeline-channel-- prefix immediately (here). + # In the downstream compiler logic, we get to treat this output like a + # normal task output. + return compiler_utils.additional_input_name_for_pipeline_channel( + f'{self.condition_branches_group.name}-oneof-{self.condition_branches_group.get_oneof_id()}' + ) + + def _validate_channels( + self, + channels: List[Union[PipelineParameterChannel, + PipelineArtifactChannel]], + ): + self._validate_no_collected_channel(channels) + self._validate_no_oneof_channel(channels) + self._validate_no_mix_of_parameters_and_artifacts(channels) + self._validate_has_else_group(self.condition_branches_group) + + def _validate_no_collected_channel( + self, channels: List[Union[PipelineParameterChannel, + PipelineArtifactChannel]] + ) -> None: + # avoid circular imports + from kfp.dsl import for_loop + if any(isinstance(channel, for_loop.Collected) for channel in channels): + raise ValueError( + f'dsl.{for_loop.Collected.__name__} cannot be used inside of dsl.{OneOf.__name__}.' + ) + + def _validate_no_oneof_channel( + self, channels: List[Union[PipelineParameterChannel, + PipelineArtifactChannel]] + ) -> None: + if any(isinstance(channel, OneOfMixin) for channel in channels): + raise ValueError( + f'dsl.{OneOf.__name__} cannot be used inside of another dsl.{OneOf.__name__}.' + ) + + def _validate_no_mix_of_parameters_and_artifacts( + self, channels: List[Union[PipelineParameterChannel, + PipelineArtifactChannel]] + ) -> None: + + first_channel = channels[0] + if isinstance(first_channel, PipelineParameterChannel): + first_channel_type = PipelineParameterChannel + else: + first_channel_type = PipelineArtifactChannel + + for channel in channels: + # if not all channels match the first channel's type, then there + # is a mix of parameter and artifact channels + if not isinstance(channel, first_channel_type): + raise TypeError( + f'Task outputs passed to dsl.{OneOf.__name__} must be the same type. Found a mix of parameters and artifacts passed to dsl.{OneOf.__name__}.' + ) + + def _validate_has_else_group( + self, + parent_group: 'tasks_group.ConditionBranches', + ) -> None: + # avoid circular imports + from kfp.dsl import tasks_group + if not isinstance(parent_group.groups[-1], tasks_group.Else): + raise ValueError( + f'dsl.{OneOf.__name__} must include an output from a task in a dsl.{tasks_group.Else.__name__} group to ensure at least one output is available at runtime.' + ) + + def __str__(self): + # supporting oneof in f-strings is technically feasible, but would + # require somehow encoding all of the oneof channels into the + # f-string + # another way to do this would be to maintain a pipeline-level + # map of PipelineChannels and encode a lookup key in the f-string + # the combination of OneOf and an f-string is not common, so prefer + # deferring implementation + raise NotImplementedError( + f'dsl.{OneOf.__name__} does not support string interpolation.') + + @property + def pattern(self) -> str: + # override self.pattern to avoid calling __str__, allowing us to block f-strings for now + # this makes it OneOfMixin hashable for use in sets/dicts + task_name = self.task_name or '' + name = self.name + channel_type = self.channel_type or '' + if isinstance(channel_type, dict): + channel_type = json.dumps(channel_type) + return _PIPELINE_CHANNEL_PLACEHOLDER_TEMPLATE % (task_name, name, + channel_type) + + +# splitting out OneOf into subclasses significantly decreases the amount of +# branching in downstream compiler logic, since the +# isinstance(, PipelineParameterChannel/PipelineArtifactChannel) +# checks continue to behave in desirable ways +class OneOfParameter(PipelineParameterChannel, OneOfMixin): + """OneOf that results in a parameter channel for all downstream tasks.""" + + def __init__(self, channels: List[PipelineParameterChannel]) -> None: + self.channels = channels + self._set_condition_branches_group(channels) + super().__init__( + name=self._make_oneof_name(), + channel_type=channels[0].channel_type, + task_name=None, + ) + self.task_name = self.condition_branches_group.name + self.channels = channels + self._validate_channels(channels) + self._validate_same_kfp_type(channels) + + def _validate_same_kfp_type( + self, channels: List[PipelineParameterChannel]) -> None: + expected_type = channels[0].channel_type + for i, channel in enumerate(channels[1:], start=1): + if channel.channel_type != expected_type: + raise TypeError( + f'Task outputs passed to dsl.{OneOf.__name__} must be the same type. Got two channels with different types: {expected_type} at index 0 and {channel.channel_type} at index {i}.' + ) + + +class OneOfArtifact(PipelineArtifactChannel, OneOfMixin): + """OneOf that results in an artifact channel for all downstream tasks.""" + + def __init__(self, channels: List[PipelineArtifactChannel]) -> None: + self.channels = channels + self._set_condition_branches_group(channels) + super().__init__( + name=self._make_oneof_name(), + channel_type=channels[0].channel_type, + task_name=None, + is_artifact_list=channels[0].is_artifact_list, + ) + self.task_name = self.condition_branches_group.name + self._validate_channels(channels) + self._validate_same_kfp_type(channels) + + def _validate_same_kfp_type( + self, channels: List[PipelineArtifactChannel]) -> None: + # Unlike for component interface type checking where anything is + # passable to Artifact, we should require the output artifacts for a + # OneOf to be the same. This reduces the complexity/ambiguity for the + # user of the actual type checking logic. What should the type checking + # behavior be if the OneOf surfaces an Artifact and a Dataset? We can + # always loosen backward compatibly in the future, so prefer starting + # conservatively. + expected_type = channels[0].channel_type + expected_is_list = channels[0].is_artifact_list + for i, channel in enumerate(channels[1:], start=1): + if channel.channel_type != expected_type or channel.is_artifact_list != expected_is_list: + raise TypeError( + f'Task outputs passed to dsl.{OneOf.__name__} must be the same type. Got two channels with different types: {expected_type} at index 0 and {channel.channel_type} at index {i}.' + ) + + +class OneOf: + """For collecting mutually exclusive outputs from conditional branches into + a single pipeline channel. + + Args: + channels: The channels to collect into a OneOf. Must be of the same type. + + Example: + :: + + @dsl.pipeline + def flip_coin_pipeline() -> str: + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return(text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return(text='Got tails!') + + # use the output from the branch that gets executed + oneof = dsl.OneOf(print_task_1.output, print_task_2.output) + + # consume it + print_and_return(text=oneof) + + # return it + return oneof + """ + + def __new__( + cls, *channels: Union[PipelineParameterChannel, PipelineArtifactChannel] + ) -> Union[OneOfParameter, OneOfArtifact]: + first_channel = channels[0] + if isinstance(first_channel, PipelineParameterChannel): + return OneOfParameter(channels=list(channels)) + elif isinstance(first_channel, PipelineArtifactChannel): + return OneOfArtifact(channels=list(channels)) + else: + raise ValueError( + f'Got unknown input to dsl.{OneOf.__name__} with type {type(first_channel)}.' + ) + + def create_pipeline_channel( name: str, channel_type: Union[str, Dict], diff --git a/sdk/python/kfp/dsl/pipeline_channel_test.py b/sdk/python/kfp/dsl/pipeline_channel_test.py new file mode 100644 index 0000000000..b0b72be083 --- /dev/null +++ b/sdk/python/kfp/dsl/pipeline_channel_test.py @@ -0,0 +1,388 @@ +# Copyright 2021 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for kfp.dsl.pipeline_channel.""" + +from typing import List +import unittest + +from absl.testing import parameterized +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Dataset +from kfp.dsl import Output +from kfp.dsl import pipeline_channel + + +class PipelineChannelTest(parameterized.TestCase): + + def test_instantiate_pipline_channel(self): + with self.assertRaisesRegex( + TypeError, "Can't instantiate abstract class PipelineChannel"): + p = pipeline_channel.PipelineChannel( + name='channel', + channel_type='String', + ) + + def test_invalid_name(self): + with self.assertRaisesRegex( + ValueError, + 'Only letters, numbers, spaces, "_", and "-" are allowed in the ' + 'name. Must begin with a letter. Got name: 123_abc'): + p = pipeline_channel.create_pipeline_channel( + name='123_abc', + channel_type='String', + ) + + def test_task_name_and_value_both_set(self): + with self.assertRaisesRegex(ValueError, + 'task_name and value cannot be both set.'): + p = pipeline_channel.create_pipeline_channel( + name='abc', + channel_type='Integer', + task_name='task1', + value=123, + ) + + def test_invalid_type(self): + with self.assertRaisesRegex(TypeError, + 'Artifact is not a parameter type.'): + p = pipeline_channel.PipelineParameterChannel( + name='channel1', + channel_type='Artifact', + ) + + with self.assertRaisesRegex(TypeError, + 'String is not an artifact type.'): + p = pipeline_channel.PipelineArtifactChannel( + name='channel1', + channel_type='String', + task_name='task1', + is_artifact_list=False, + ) + + @parameterized.parameters( + { + 'pipeline_channel': + pipeline_channel.create_pipeline_channel( + name='channel1', + task_name='task1', + channel_type='String', + ), + 'str_repr': + '{{channel:task=task1;name=channel1;type=String;}}', + }, + { + 'pipeline_channel': + pipeline_channel.create_pipeline_channel( + name='channel2', + channel_type='Integer', + ), + 'str_repr': + '{{channel:task=;name=channel2;type=Integer;}}', + }, + { + 'pipeline_channel': + pipeline_channel.create_pipeline_channel( + name='channel3', + channel_type={'type_a': { + 'property_b': 'c' + }}, + task_name='task3', + ), + 'str_repr': + '{{channel:task=task3;name=channel3;type={"type_a": {"property_b": "c"}};}}', + }, + { + 'pipeline_channel': + pipeline_channel.create_pipeline_channel( + name='channel4', + channel_type='Float', + value=1.23, + ), + 'str_repr': + '{{channel:task=;name=channel4;type=Float;}}', + }, + { + 'pipeline_channel': + pipeline_channel.create_pipeline_channel( + name='channel5', + channel_type='system.Artifact@0.0.1', + task_name='task5', + ), + 'str_repr': + '{{channel:task=task5;name=channel5;type=system.Artifact@0.0.1;}}', + }, + ) + def test_str_repr(self, pipeline_channel, str_repr): + self.assertEqual(str_repr, str(pipeline_channel)) + + def test_extract_pipeline_channels(self): + p1 = pipeline_channel.create_pipeline_channel( + name='channel1', + channel_type='String', + value='abc', + ) + p2 = pipeline_channel.create_pipeline_channel( + name='channel2', + channel_type='customized_type_b', + task_name='task2', + ) + p3 = pipeline_channel.create_pipeline_channel( + name='channel3', + channel_type={'customized_type_c': { + 'property_c': 'value_c' + }}, + task_name='task3', + ) + stuff_chars = ' between ' + payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3) + params = pipeline_channel.extract_pipeline_channels_from_string(payload) + self.assertListEqual([p1, p2, p3], params) + + # Expecting the extract_pipelineparam_from_any to dedup pipeline channels + # among all the payloads. + payload = [ + str(p1) + stuff_chars + str(p2), + str(p2) + stuff_chars + str(p3) + ] + params = pipeline_channel.extract_pipeline_channels_from_any(payload) + self.assertListEqual([p1, p2, p3], params) + + +@dsl.component +def string_comp() -> str: + return 'text' + + +@dsl.component +def list_comp() -> List[str]: + return ['text'] + + +@dsl.component +def roll_three_sided_die() -> str: + import random + val = random.randint(0, 2) + + if val == 0: + return 'heads' + elif val == 1: + return 'tails' + else: + return 'draw' + + +@dsl.component +def print_and_return(text: str) -> str: + print(text) + return text + + +class TestCanAccessTask(unittest.TestCase): + + def test(self): + + @dsl.pipeline + def my_pipeline(): + op1 = string_comp() + self.assertEqual(op1.output.task, op1) + + +class TestOneOfAndCollectedNotComposable(unittest.TestCase): + + def test_collected_in_oneof(self): + with self.assertRaisesRegex( + ValueError, + 'dsl.Collected cannot be used inside of dsl.OneOf.'): + + @dsl.pipeline + def my_pipeline(x: str): + with dsl.If(x == 'foo'): + t1 = list_comp() + with dsl.Else(): + with dsl.ParallelFor([1, 2, 3]): + t2 = string_comp() + collected = dsl.Collected(t2.output) + # test cases doesn't return or pass to task to ensure validation is in the OneOf + dsl.OneOf(t1.output, collected) + + def test_oneof_in_collected(self): + with self.assertRaisesRegex( + ValueError, + 'dsl.OneOf cannot be used inside of dsl.Collected.'): + + @dsl.pipeline + def my_pipeline(x: str): + with dsl.ParallelFor([1, 2, 3]): + with dsl.If(x == 'foo'): + t1 = string_comp() + with dsl.Else(): + t2 = string_comp() + oneof = dsl.OneOf(t1.output, t2.output) + # test cases doesn't return or pass to task to ensure validation is in the Collected constructor + dsl.Collected(oneof) + + +class TestOneOfRequiresSameType(unittest.TestCase): + + def test_same_parameter_type(self): + + @dsl.pipeline + def my_pipeline(x: str) -> str: + with dsl.If(x == 'foo'): + t1 = string_comp() + with dsl.Else(): + t2 = string_comp() + return dsl.OneOf(t1.output, t2.output) + + self.assertEqual( + my_pipeline.pipeline_spec.components['comp-condition-branches-1'] + .output_definitions.parameters[ + 'pipelinechannel--condition-branches-1-oneof-1'].parameter_type, + 3) + + def test_different_parameter_types(self): + + with self.assertRaisesRegex( + TypeError, + r'Task outputs passed to dsl\.OneOf must be the same type. Got two channels with different types: String at index 0 and typing\.List\[str\] at index 1\.' + ): + + @dsl.pipeline + def my_pipeline(x: str) -> str: + with dsl.If(x == 'foo'): + t1 = string_comp() + with dsl.Else(): + t2 = list_comp() + return dsl.OneOf(t1.output, t2.output) + + def test_same_artifact_type(self): + + @dsl.component + def artifact_comp(out: Output[Artifact]): + with open(out.path, 'w') as f: + f.write('foo') + + @dsl.pipeline + def my_pipeline(x: str) -> Artifact: + with dsl.If(x == 'foo'): + t1 = artifact_comp() + with dsl.Else(): + t2 = artifact_comp() + return dsl.OneOf(t1.outputs['out'], t2.outputs['out']) + + self.assertEqual( + my_pipeline.pipeline_spec.components['comp-condition-branches-1'] + .output_definitions + .artifacts['pipelinechannel--condition-branches-1-oneof-1'] + .artifact_type.schema_title, + 'system.Artifact', + ) + self.assertEqual( + my_pipeline.pipeline_spec.components['comp-condition-branches-1'] + .output_definitions + .artifacts['pipelinechannel--condition-branches-1-oneof-1'] + .artifact_type.schema_version, + '0.0.1', + ) + + def test_different_artifact_type(self): + + @dsl.component + def artifact_comp_one(out: Output[Artifact]): + with open(out.path, 'w') as f: + f.write('foo') + + @dsl.component + def artifact_comp_two(out: Output[Dataset]): + with open(out.path, 'w') as f: + f.write('foo') + + with self.assertRaisesRegex( + TypeError, + r'Task outputs passed to dsl\.OneOf must be the same type. Got two channels with different types: system.Artifact@0.0.1 at index 0 and system.Dataset@0.0.1 at index 1\.' + ): + + @dsl.pipeline + def my_pipeline(x: str) -> Artifact: + with dsl.If(x == 'foo'): + t1 = artifact_comp_one() + with dsl.Else(): + t2 = artifact_comp_two() + return dsl.OneOf(t1.outputs['out'], t2.outputs['out']) + + def test_different_artifact_type_due_to_list(self): + # if we ever support list of artifact outputs from components, this test will fail, which is good because it needs to be changed + + with self.assertRaisesRegex( + ValueError, + r"Output lists of artifacts are only supported for pipelines\. Got output list of artifacts for output parameter 'out' of component 'artifact-comp-two'\." + ): + + @dsl.component + def artifact_comp_one(out: Output[Artifact]): + with open(out.path, 'w') as f: + f.write('foo') + + @dsl.component + def artifact_comp_two(out: Output[List[Artifact]]): + with open(out.path, 'w') as f: + f.write('foo') + + @dsl.pipeline + def my_pipeline(x: str) -> Artifact: + with dsl.If(x == 'foo'): + t1 = artifact_comp_one() + with dsl.Else(): + t2 = artifact_comp_two() + return dsl.OneOf(t1.outputs['out'], t2.outputs['out']) + + def test_parameters_mixed_with_artifacts(self): + + @dsl.component + def artifact_comp(out: Output[Artifact]): + with open(out.path, 'w') as f: + f.write('foo') + + with self.assertRaisesRegex( + TypeError, + r'Task outputs passed to dsl\.OneOf must be the same type\. Found a mix of parameters and artifacts passed to dsl\.OneOf\.' + ): + + @dsl.pipeline + def my_pipeline(x: str) -> str: + with dsl.If(x == 'foo'): + t1 = artifact_comp() + with dsl.Else(): + t2 = string_comp() + return dsl.OneOf(t1.output, t2.output) + + def test_no_else_raises(self): + with self.assertRaisesRegex( + ValueError, + r'dsl\.OneOf must include an output from a task in a dsl\.Else group to ensure at least one output is available at runtime\.' + ): + + @dsl.pipeline + def roll_die_pipeline(): + flip_coin_task = roll_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + print_and_return(text=dsl.OneOf(t1.output, t2.output)) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdk/python/kfp/components/pipeline_context.py b/sdk/python/kfp/dsl/pipeline_context.py similarity index 91% rename from sdk/python/kfp/components/pipeline_context.py rename to sdk/python/kfp/dsl/pipeline_context.py index a41e401da6..4881bc5680 100644 --- a/sdk/python/kfp/components/pipeline_context.py +++ b/sdk/python/kfp/dsl/pipeline_context.py @@ -16,10 +16,10 @@ import functools from typing import Callable, Optional -from kfp.components import component_factory -from kfp.components import pipeline_task -from kfp.components import tasks_group -from kfp.components import utils +from kfp.dsl import component_factory +from kfp.dsl import pipeline_task +from kfp.dsl import tasks_group +from kfp.dsl import utils def pipeline(func: Optional[Callable] = None, @@ -182,6 +182,7 @@ def push_tasks_group(self, group: 'tasks_group.TasksGroup'): group: A TasksGroup. Typically it is one of ExitHandler, Condition, and ParallelFor. """ + group.parent_task_group = self.get_parent_group() self.groups[-1].groups.append(group) self.groups.append(group) @@ -189,6 +190,15 @@ def pop_tasks_group(self): """Removes the current TasksGroup from the stack.""" del self.groups[-1] + def get_last_tasks_group(self) -> Optional['tasks_group.TasksGroup']: + """Gets the last TasksGroup added to the pipeline at the current level + of the pipeline definition.""" + groups = self.groups[-1].groups + return groups[-1] if groups else None + + def get_parent_group(self) -> 'tasks_group.TasksGroup': + return self.groups[-1] + def remove_task_from_groups(self, task: pipeline_task.PipelineTask): """Removes a task from the pipeline. diff --git a/sdk/python/kfp/components/pipeline_task.py b/sdk/python/kfp/dsl/pipeline_task.py similarity index 97% rename from sdk/python/kfp/components/pipeline_task.py rename to sdk/python/kfp/dsl/pipeline_task.py index f1e47bbaf1..ecf1640a88 100644 --- a/sdk/python/kfp/components/pipeline_task.py +++ b/sdk/python/kfp/dsl/pipeline_task.py @@ -20,12 +20,12 @@ from typing import Any, Dict, List, Mapping, Optional, Union import warnings -from kfp.components import constants -from kfp.components import pipeline_channel -from kfp.components import placeholders -from kfp.components import structures -from kfp.components import utils -from kfp.components.types import type_utils +from kfp.dsl import constants +from kfp.dsl import pipeline_channel +from kfp.dsl import placeholders +from kfp.dsl import structures +from kfp.dsl import utils +from kfp.dsl.types import type_utils from kfp.pipeline_spec import pipeline_spec_pb2 _register_task_handler = lambda task: utils.maybe_rename_for_k8s( @@ -69,7 +69,7 @@ def __init__( ): """Initilizes a PipelineTask instance.""" # import within __init__ to avoid circular import - from kfp.components.tasks_group import TasksGroup + from kfp.dsl.tasks_group import TasksGroup self.parent_task_group: Union[None, TasksGroup] = None args = args or {} @@ -604,6 +604,8 @@ def my_pipeline(text: str = 'message'): for input_spec_name, input_spec in (self.component_spec.inputs or {}).items(): + if type_utils.is_task_final_status_type(input_spec.type): + continue argument_value = self._inputs[input_spec_name] if (isinstance(argument_value, pipeline_channel.PipelineChannel) ) and (not input_spec.optional) and (argument_value.task_name @@ -617,7 +619,7 @@ def my_pipeline(text: str = 'message'): return self -# TODO: this function should ideally be in the function kfp.components.structures.check_placeholder_references_valid_io_name, which does something similar, but this causes the exception to be raised at component definition time, rather than compile time. This would break tests that load v1 component YAML, even though that YAML is invalid. +# TODO: this function should ideally be in the function kfp.dsl.structures.check_placeholder_references_valid_io_name, which does something similar, but this causes the exception to be raised at component definition time, rather than compile time. This would break tests that load v1 component YAML, even though that YAML is invalid. def check_primitive_placeholder_is_used_for_correct_io_type( inputs_dict: Dict[str, structures.InputSpec], outputs_dict: Dict[str, structures.OutputSpec], diff --git a/sdk/python/kfp/components/pipeline_task_test.py b/sdk/python/kfp/dsl/pipeline_task_test.py similarity index 98% rename from sdk/python/kfp/components/pipeline_task_test.py rename to sdk/python/kfp/dsl/pipeline_task_test.py index 128a83a349..6e7443fc1a 100644 --- a/sdk/python/kfp/components/pipeline_task_test.py +++ b/sdk/python/kfp/dsl/pipeline_task_test.py @@ -11,16 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for kfp.components.pipeline_task.""" +"""Tests for kfp.dsl.pipeline_task.""" import textwrap import unittest from absl.testing import parameterized from kfp import dsl -from kfp.components import pipeline_task -from kfp.components import placeholders -from kfp.components import structures +from kfp.dsl import pipeline_task +from kfp.dsl import placeholders +from kfp.dsl import structures V2_YAML = textwrap.dedent("""\ components: diff --git a/sdk/python/kfp/components/placeholders.py b/sdk/python/kfp/dsl/placeholders.py similarity index 99% rename from sdk/python/kfp/components/placeholders.py rename to sdk/python/kfp/dsl/placeholders.py index 701f463ada..39a2617cff 100644 --- a/sdk/python/kfp/components/placeholders.py +++ b/sdk/python/kfp/dsl/placeholders.py @@ -18,8 +18,8 @@ import json from typing import Any, Dict, List, Optional, Union -from kfp.components import utils -from kfp.components.types import type_utils +from kfp.dsl import utils +from kfp.dsl.types import type_utils class Placeholder(abc.ABC): diff --git a/sdk/python/kfp/components/placeholders_test.py b/sdk/python/kfp/dsl/placeholders_test.py similarity index 99% rename from sdk/python/kfp/components/placeholders_test.py rename to sdk/python/kfp/dsl/placeholders_test.py index 9c89660621..da89f4cfa8 100644 --- a/sdk/python/kfp/components/placeholders_test.py +++ b/sdk/python/kfp/dsl/placeholders_test.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Contains tests for kfp.components.placeholders.""" +"""Contains tests for kfp.dsl.placeholders.""" import os import tempfile from typing import Any, List @@ -19,11 +19,11 @@ from absl.testing import parameterized from kfp import compiler from kfp import dsl -from kfp.components import placeholders from kfp.dsl import Artifact from kfp.dsl import Dataset from kfp.dsl import Input from kfp.dsl import Output +from kfp.dsl import placeholders class TestExecutorInputPlaceholder(parameterized.TestCase): diff --git a/sdk/python/kfp/components/python_component.py b/sdk/python/kfp/dsl/python_component.py similarity index 92% rename from sdk/python/kfp/components/python_component.py rename to sdk/python/kfp/dsl/python_component.py index 8106f2e4bf..faa4c44740 100644 --- a/sdk/python/kfp/components/python_component.py +++ b/sdk/python/kfp/dsl/python_component.py @@ -15,11 +15,11 @@ from typing import Callable -from kfp import components -from kfp.components import structures +from kfp.dsl import base_component +from kfp.dsl import structures -class PythonComponent(components.BaseComponent): +class PythonComponent(base_component.BaseComponent): """A component defined via Python function. **Note:** ``PythonComponent`` is not intended to be used to construct components directly. Use ``@kfp.dsl.component`` instead. diff --git a/sdk/python/kfp/components/structures.py b/sdk/python/kfp/dsl/structures.py similarity index 97% rename from sdk/python/kfp/components/structures.py rename to sdk/python/kfp/dsl/structures.py index a39cff4fee..4b4e6fba1c 100644 --- a/sdk/python/kfp/components/structures.py +++ b/sdk/python/kfp/dsl/structures.py @@ -23,15 +23,14 @@ from google.protobuf import json_format import kfp -from kfp.components import placeholders -from kfp.components import utils -from kfp.components import v1_components -from kfp.components import v1_structures -from kfp.components.container_component_artifact_channel import \ +from kfp.dsl import placeholders +from kfp.dsl import utils +from kfp.dsl import v1_structures +from kfp.dsl.container_component_artifact_channel import \ ContainerComponentArtifactChannel -from kfp.components.types import artifact_types -from kfp.components.types import type_annotations -from kfp.components.types import type_utils +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_annotations +from kfp.dsl.types import type_utils from kfp.pipeline_spec import pipeline_spec_pb2 import yaml @@ -668,8 +667,8 @@ def from_v1_component_spec( continue elif isinstance(type_, str) and type_.lower( - ) in type_utils._PARAMETER_TYPES_MAPPING: - type_enum = type_utils._PARAMETER_TYPES_MAPPING[type_.lower()] + ) in type_utils.PARAMETER_TYPES_MAPPING: + type_enum = type_utils.PARAMETER_TYPES_MAPPING[type_.lower()] ir_parameter_type_name = pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( type_enum) in_memory_parameter_type_name = type_utils.IR_TYPE_TO_IN_MEMORY_SPEC_TYPE[ @@ -721,8 +720,8 @@ def from_v1_component_spec( type_ = type_utils.get_canonical_name_for_outer_generic(type_) if isinstance(type_, str) and type_.lower( - ) in type_utils._PARAMETER_TYPES_MAPPING: - type_enum = type_utils._PARAMETER_TYPES_MAPPING[type_.lower()] + ) in type_utils.PARAMETER_TYPES_MAPPING: + type_enum = type_utils.PARAMETER_TYPES_MAPPING[type_.lower()] ir_parameter_type_name = pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( type_enum) in_memory_parameter_type_name = type_utils.IR_TYPE_TO_IN_MEMORY_SPEC_TYPE[ @@ -798,7 +797,7 @@ def outputs_dict_from_component_spec_dict( } def extract_description_from_command( - commands: List[str]) -> Union[str, None]: + commands: List[str]) -> Optional[str]: for command in commands: if isinstance(command, str) and 'import kfp' in command: for node in ast.walk(ast.parse(command)): @@ -847,7 +846,7 @@ def from_yaml_documents(cls, component_yaml: str) -> 'ComponentSpec': ComponentSpec: The ComponentSpec object. """ - def extract_description(component_yaml: str) -> Union[str, None]: + def extract_description(component_yaml: str) -> Optional[str]: heading = '# Description: ' multi_line_description_prefix = '# ' index_of_heading = 2 @@ -872,7 +871,7 @@ def extract_description(component_yaml: str) -> Union[str, None]: is_v1 = 'implementation' in set(pipeline_spec_dict.keys()) if is_v1: - v1_component = v1_components._load_component_spec_from_component_text( + v1_component = _load_component_spec_from_component_text( component_yaml) return cls.from_v1_component_spec(v1_component) else: @@ -912,9 +911,9 @@ def to_pipeline_spec(self) -> pipeline_spec_pb2.PipelineSpec: # import here to aviod circular module dependency from kfp.compiler import compiler_utils from kfp.compiler import pipeline_spec_builder as builder - from kfp.components import pipeline_channel - from kfp.components import pipeline_task - from kfp.components import tasks_group + from kfp.dsl import pipeline_channel + from kfp.dsl import pipeline_task + from kfp.dsl import tasks_group args_dict = {} pipeline_inputs = self.inputs or {} @@ -1073,3 +1072,15 @@ def load_documents_from_yaml(component_yaml: str) -> Tuple[dict, dict]: f'Expected one or two YAML documents in the IR YAML file. Got: {num_docs}.' ) return pipeline_spec_dict, platform_spec_dict + + +def _load_component_spec_from_component_text( + text) -> v1_structures.ComponentSpec: + component_dict = yaml.safe_load(text) + component_spec = v1_structures.ComponentSpec.from_dict(component_dict) + + # Calculating hash digest for the component + data = text if isinstance(text, bytes) else text.encode('utf-8') + data = data.replace(b'\r\n', b'\n') # Normalizing line endings + + return component_spec diff --git a/sdk/python/kfp/components/structures_test.py b/sdk/python/kfp/dsl/structures_test.py similarity index 99% rename from sdk/python/kfp/components/structures_test.py rename to sdk/python/kfp/dsl/structures_test.py index 6bacae4e77..ad6274d931 100644 --- a/sdk/python/kfp/components/structures_test.py +++ b/sdk/python/kfp/dsl/structures_test.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for kfp.components.structures.""" +"""Tests for kfp.dsl.structures.""" import os import tempfile @@ -22,9 +22,9 @@ from kfp import compiler from kfp import components from kfp import dsl -from kfp.components import component_factory -from kfp.components import placeholders -from kfp.components import structures +from kfp.dsl import component_factory +from kfp.dsl import placeholders +from kfp.dsl import structures V1_YAML_IF_PLACEHOLDER = textwrap.dedent("""\ implementation: diff --git a/sdk/python/kfp/components/task_final_status.py b/sdk/python/kfp/dsl/task_final_status.py similarity index 100% rename from sdk/python/kfp/components/task_final_status.py rename to sdk/python/kfp/dsl/task_final_status.py diff --git a/sdk/python/kfp/dsl/tasks_group.py b/sdk/python/kfp/dsl/tasks_group.py new file mode 100644 index 0000000000..3f0f758bbd --- /dev/null +++ b/sdk/python/kfp/dsl/tasks_group.py @@ -0,0 +1,473 @@ +# Copyright 2021 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Definition for TasksGroup.""" + +import copy +import enum +from typing import List, Optional, Union +import warnings + +from kfp.dsl import for_loop +from kfp.dsl import pipeline_channel +from kfp.dsl import pipeline_context +from kfp.dsl import pipeline_task + + +class TasksGroupType(str, enum.Enum): + """Types of TasksGroup.""" + PIPELINE = 'pipeline' + CONDITION = 'condition' + CONDITION_BRANCHES = 'condition-branches' + FOR_LOOP = 'for-loop' + EXIT_HANDLER = 'exit-handler' + + +class TasksGroup: + """Represents a logical group of tasks and groups of TasksGroups. + + This class is the base class for groups of tasks, such as tasks + sharing an exit handler, a condition branch, or a loop. This class + is not supposed to be used by pipeline authors. It is useful for + implementing a compiler. + + Attributes: + group_type: The type of the TasksGroup. + tasks: A list of all PipelineTasks in this group. + groups: A list of TasksGroups in this group. + display_name: The optional user given name of the group. + dependencies: A list of tasks or groups this group depends on. + is_root: If TasksGroup is root group. + """ + + def __init__( + self, + group_type: TasksGroupType, + name: Optional[str] = None, + is_root: bool = False, + ) -> None: + """Create a new instance of TasksGroup. + + Args: + group_type: The type of the group. + name: The name of the group. Used as display name in UI. + """ + self.group_type = group_type + self.tasks = [] + self.groups = [] + self.display_name = name + self.dependencies = [] + self.is_root = is_root + # backref to parent, set when the pipeline is called in pipeline_context + self.parent_task_group: Optional[TasksGroup] = None + + def __enter__(self): + if not pipeline_context.Pipeline.get_default_pipeline(): + raise ValueError('Default pipeline not defined.') + + self._make_name_unique() + + pipeline_context.Pipeline.get_default_pipeline().push_tasks_group(self) + return self + + def __exit__(self, *unused_args): + pipeline_context.Pipeline.get_default_pipeline().pop_tasks_group() + + def _make_name_unique(self): + """Generates a unique TasksGroup name in the pipeline.""" + if not pipeline_context.Pipeline.get_default_pipeline(): + raise ValueError('Default pipeline not defined.') + + group_id = pipeline_context.Pipeline.get_default_pipeline( + ).get_next_group_id() + self.name = f'{self.group_type.value}-{group_id}' + self.name = self.name.replace('_', '-') + + def remove_task_recursive(self, task: pipeline_task.PipelineTask): + """Removes a task from the group recursively.""" + if self.tasks and task in self.tasks: + self.tasks.remove(task) + for group in self.groups or []: + group.remove_task_recursive(task) + + +class ExitHandler(TasksGroup): + """A class for setting an exit handler task that is invoked upon exiting a + group of other tasks. + + Args: + exit_task: The task that is invoked after exiting a group of other tasks. + name: The name of the exit handler group. + + Example: + :: + + exit_task = ExitComponent(...) + with ExitHandler(exit_task): + task1 = my_component1(...) + task2 = my_component2(...) + """ + + def __init__( + self, + exit_task: pipeline_task.PipelineTask, + name: Optional[str] = None, + ) -> None: + """Initializes a Condition task group.""" + super().__init__( + group_type=TasksGroupType.EXIT_HANDLER, + name=name, + is_root=False, + ) + + if exit_task.dependent_tasks: + raise ValueError('exit_task cannot depend on any other tasks.') + + # Removing exit_task form any group + pipeline_context.Pipeline.get_default_pipeline( + ).remove_task_from_groups(exit_task) + + # Set is_exit_handler since the compiler might be using this attribute. + exit_task.is_exit_handler = True + + self.exit_task = exit_task + + +class ConditionBranches(TasksGroup): + _oneof_id = 0 + + def __init__(self) -> None: + super().__init__( + group_type=TasksGroupType.CONDITION_BRANCHES, + name=None, + is_root=False, + ) + + def get_oneof_id(self) -> int: + """Incrementor for uniquely identifying a OneOf for the parent + ConditionBranches group. + + This is analogous to incrementing a unique identifier for tasks + groups belonging to a pipeline. + """ + self._oneof_id += 1 + return self._oneof_id + + +class _ConditionBase(TasksGroup): + """Parent class for condition control flow context managers (Condition, If, + Elif, Else). + + Args: + condition: A list of binary operations to be combined via conjunction. + name: The name of the condition group. + """ + + def __init__( + self, + conditions: List[pipeline_channel.ConditionOperation], + name: Optional[str] = None, + ) -> None: + super().__init__( + group_type=TasksGroupType.CONDITION, + name=name, + is_root=False, + ) + self.conditions: List[pipeline_channel.ConditionOperation] = conditions + + +class If(_ConditionBase): + """A class for creating a conditional control flow "if" block within a + pipeline. + + Args: + condition: A comparative expression that evaluates to True or False. At least one of the operands must be an output from an upstream task or a pipeline parameter. + name: The name of the condition group. + + Example: + :: + + task1 = my_component1(...) + with dsl.If(task1.output=='pizza', 'pizza-condition'): + task2 = my_component2(...) + """ + + def __init__( + self, + condition, + name: Optional[str] = None, + ) -> None: + super().__init__( + conditions=[condition], + name=name, + ) + if isinstance(condition, bool): + raise ValueError( + f'Got constant boolean {condition} as a condition. This is likely because the provided condition evaluated immediately. At least one of the operands must be an output from an upstream task or a pipeline parameter.' + ) + copied_condition = copy.copy(condition) + copied_condition.negate = True + self._negated_upstream_conditions = [copied_condition] + + +class Condition(If): + """Deprecated. + + Use dsl.If instead. + """ + + def __enter__(self): + super().__enter__() + warnings.warn( + 'dsl.Condition is deprecated. Please use dsl.If instead.', + category=DeprecationWarning, + stacklevel=2) + return self + + +class Elif(_ConditionBase): + """A class for creating a conditional control flow "else if" block within a + pipeline. Can be used following an upstream dsl.If or dsl.Elif. + + Args: + condition: A comparative expression that evaluates to True or False. At least one of the operands must be an output from an upstream task or a pipeline parameter. + name: The name of the condition group. + + Example: + :: + + task1 = my_component1(...) + task2 = my_component2(...) + with dsl.If(task1.output=='pizza', 'pizza-condition'): + task3 = my_component3(...) + + with dsl.Elif(task2.output=='pasta', 'pasta-condition'): + task4 = my_component4(...) + """ + + def __init__( + self, + condition, + name: Optional[str] = None, + ) -> None: + prev_cond = pipeline_context.Pipeline.get_default_pipeline( + ).get_last_tasks_group() + if not isinstance(prev_cond, (Condition, If, Elif)): + # prefer pushing toward dsl.If rather than dsl.Condition for syntactic consistency with the if-elif-else keywords in Python + raise InvalidControlFlowException( + 'dsl.Elif can only be used following an upstream dsl.If or dsl.Elif.' + ) + + if isinstance(condition, bool): + raise ValueError( + f'Got constant boolean {condition} as a condition. This is likely because the provided condition evaluated immediately. At least one of the operands must be an output from an upstream task or a pipeline parameter.' + ) + + copied_condition = copy.copy(condition) + copied_condition.negate = True + self._negated_upstream_conditions = _shallow_copy_list_of_binary_operations( + prev_cond._negated_upstream_conditions) + [copied_condition] + + conditions = _shallow_copy_list_of_binary_operations( + prev_cond._negated_upstream_conditions) + conditions.append(condition) + + super().__init__( + conditions=conditions, + name=name, + ) + + def __enter__(self): + if not pipeline_context.Pipeline.get_default_pipeline(): + raise ValueError('Default pipeline not defined.') + + pipeline = pipeline_context.Pipeline.get_default_pipeline() + + maybe_make_and_insert_conditional_branches_group(pipeline) + + self._make_name_unique() + pipeline.push_tasks_group(self) + return self + + +class Else(_ConditionBase): + """A class for creating a conditional control flow "else" block within a + pipeline. Can be used following an upstream dsl.If or dsl.Elif. + + Args: + name: The name of the condition group. + + Example: + :: + + task1 = my_component1(...) + task2 = my_component2(...) + with dsl.If(task1.output=='pizza', 'pizza-condition'): + task3 = my_component3(...) + + with dsl.Elif(task2.output=='pasta', 'pasta-condition'): + task4 = my_component4(...) + + with dsl.Else(): + my_component5(...) + """ + + def __init__( + self, + name: Optional[str] = None, + ) -> None: + prev_cond = pipeline_context.Pipeline.get_default_pipeline( + ).get_last_tasks_group() + + # if it immediately follows as TasksGroup, this is because it immediately + # follows Else in the user code and we wrap Else in a TasksGroup + if isinstance(prev_cond, ConditionBranches): + # prefer pushing toward dsl.If rather than dsl.Condition for syntactic consistency with the if-elif-else keywords in Python + raise InvalidControlFlowException( + 'Cannot use dsl.Else following another dsl.Else. dsl.Else can only be used following an upstream dsl.If or dsl.Elif.' + ) + if not isinstance(prev_cond, (Condition, If, Elif)): + # prefer pushing toward dsl.If rather than dsl.Condition for syntactic consistency with the if-elif-else keywords in Python + raise InvalidControlFlowException( + 'dsl.Else can only be used following an upstream dsl.If or dsl.Elif.' + ) + + super().__init__( + conditions=prev_cond._negated_upstream_conditions, + name=name, + ) + + def __enter__(self): + if not pipeline_context.Pipeline.get_default_pipeline(): + raise ValueError('Default pipeline not defined.') + + pipeline = pipeline_context.Pipeline.get_default_pipeline() + + maybe_make_and_insert_conditional_branches_group(pipeline) + + self._make_name_unique() + pipeline.push_tasks_group(self) + return self + + def __exit__(self, *unused_args): + pipeline = pipeline_context.Pipeline.get_default_pipeline() + pipeline.pop_tasks_group() + + # since this is an else, also pop off the parent dag for conditional branches + # this parent TasksGroup is not a context manager, so we simulate its + # __exit__ call with this + pipeline.pop_tasks_group() + + +def maybe_make_and_insert_conditional_branches_group( + pipeline: 'pipeline_context.Pipeline') -> None: + + already_has_pipeline_wrapper = isinstance( + pipeline.get_last_tasks_group(), + Elif, + ) + if already_has_pipeline_wrapper: + return + + condition_wrapper_group = ConditionBranches() + condition_wrapper_group._make_name_unique() + + # swap outer and inner group ids so that numbering stays sequentially consistent with how such hypothetical code would be authored + def swap_group_ids(parent: TasksGroup, cond: TasksGroup): + parent_name, parent_id = parent.name.rsplit('-', 1) + cond_name, cond_id = cond.name.split('-') + cond.name = f'{cond_name}-{parent_id}' + parent.name = f'{parent_name}-{cond_id}' + + # replace last pushed group (If or Elif) with condition group + last_pushed_group = pipeline.groups[-1].groups.pop() + swap_group_ids(condition_wrapper_group, last_pushed_group) + pipeline.push_tasks_group(condition_wrapper_group) + + # then repush (__enter__) and pop (__exit__) the last pushed group + # before the wrapper to emulate re-entering and exiting its context + pipeline.push_tasks_group(last_pushed_group) + pipeline.pop_tasks_group() + + +class InvalidControlFlowException(Exception): + pass + + +def _shallow_copy_list_of_binary_operations( + operations: List[pipeline_channel.ConditionOperation] +) -> List[pipeline_channel.ConditionOperation]: + # shallow copy is sufficient to allow us to invert the negate flag of a ConditionOperation without affecting copies. deep copy not needed and would result in many copies of the full pipeline since PipelineChannels hold references to the pipeline. + return [copy.copy(operation) for operation in operations] + + +class ParallelFor(TasksGroup): + """A class for creating parallelized for loop control flow over a static + set of items within a pipeline definition. + + Args: + items: The items to loop over. It can be either a constant Python list or a list output from an upstream task. + name: The name of the for loop group. + parallelism: The maximum number of concurrent iterations that can be scheduled for execution. A value of 0 represents unconstrained parallelism (default is unconstrained). + + Example: + :: + + with dsl.ParallelFor( + items=[{'a': 1, 'b': 10}, {'a': 2, 'b': 20}], + parallelism=1 + ) as item: + task1 = my_component(..., number=item.a) + task2 = my_component(..., number=item.b) + + In the example, the group of tasks containing ``task1`` and ``task2`` would + be executed twice, once with case ``args=[{'a': 1, 'b': 10}]`` and once with + case ``args=[{'a': 2, 'b': 20}]``. The ``parallelism=1`` setting causes only + 1 execution to be scheduled at a time. + """ + + def __init__( + self, + items: Union[for_loop.ItemList, pipeline_channel.PipelineChannel], + name: Optional[str] = None, + parallelism: Optional[int] = None, + ) -> None: + """Initializes a for loop task group.""" + parallelism = parallelism or 0 + if parallelism < 0: + raise ValueError( + f'ParallelFor parallelism must be >= 0. Got: {parallelism}.') + + super().__init__( + group_type=TasksGroupType.FOR_LOOP, + name=name, + is_root=False, + ) + + if isinstance(items, pipeline_channel.PipelineChannel): + self.loop_argument = for_loop.LoopArgument.from_pipeline_channel( + items) + self.items_is_pipeline_channel = True + else: + self.loop_argument = for_loop.LoopArgument.from_raw_items( + raw_items=items, + name_code=pipeline_context.Pipeline.get_default_pipeline() + .get_next_group_id(), + ) + self.items_is_pipeline_channel = False + + self.parallelism_limit = parallelism + + def __enter__(self) -> for_loop.LoopArgument: + super().__enter__() + return self.loop_argument diff --git a/sdk/python/kfp/components/tasks_group_test.py b/sdk/python/kfp/dsl/tasks_group_test.py similarity index 80% rename from sdk/python/kfp/components/tasks_group_test.py rename to sdk/python/kfp/dsl/tasks_group_test.py index c64346b245..40c68ab372 100644 --- a/sdk/python/kfp/components/tasks_group_test.py +++ b/sdk/python/kfp/dsl/tasks_group_test.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from absl.testing import parameterized -from kfp.components import for_loop -from kfp.components import pipeline_context -from kfp.components import tasks_group +import unittest +from kfp import dsl +from kfp.dsl import for_loop +from kfp.dsl import pipeline_context +from kfp.dsl import tasks_group -class ParallelForTest(parameterized.TestCase): + +class ParallelForTest(unittest.TestCase): def test_basic(self): loop_items = ['pizza', 'hotdog', 'pasta'] @@ -58,3 +60,21 @@ def test_parallelfor_invalid_parallelism(self): 'ParallelFor parallelism must be >= 0.'): with pipeline_context.Pipeline('pipeline') as p: tasks_group.ParallelFor(items=loop_items, parallelism=-1) + + +class TestConditionDeprecated(unittest.TestCase): + + def test(self): + + @dsl.component + def foo() -> str: + return 'foo' + + @dsl.pipeline + def my_pipeline(string: str): + with self.assertWarnsRegex( + DeprecationWarning, + 'dsl\.Condition is deprecated\. Please use dsl\.If instead\.' + ): + with dsl.Condition(string == 'text'): + foo() diff --git a/sdk/python/kfp/components/types/__init__.py b/sdk/python/kfp/dsl/types/__init__.py similarity index 100% rename from sdk/python/kfp/components/types/__init__.py rename to sdk/python/kfp/dsl/types/__init__.py diff --git a/sdk/python/kfp/components/types/artifact_types.py b/sdk/python/kfp/dsl/types/artifact_types.py similarity index 88% rename from sdk/python/kfp/components/types/artifact_types.py rename to sdk/python/kfp/dsl/types/artifact_types.py index 163e3feec3..f7a676573d 100644 --- a/sdk/python/kfp/components/types/artifact_types.py +++ b/sdk/python/kfp/dsl/types/artifact_types.py @@ -11,12 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Classes for input/output Artifacts in KFP SDK. - -These are only compatible with v2 Pipelines. -""" +"""Classes and utilities for using and creating artifacts in components.""" +import os from typing import Dict, List, Optional, Type +import warnings _GCS_LOCAL_MOUNT_PREFIX = '/gcs/' _MINIO_LOCAL_MOUNT_PREFIX = '/minio/' @@ -93,13 +92,17 @@ def _get_path(self) -> Optional[str]: return None def _set_path(self, path: str) -> None: - if path.startswith(_GCS_LOCAL_MOUNT_PREFIX): - path = 'gs://' + path[len(_GCS_LOCAL_MOUNT_PREFIX):] - elif path.startswith(_MINIO_LOCAL_MOUNT_PREFIX): - path = 'minio://' + path[len(_MINIO_LOCAL_MOUNT_PREFIX):] - elif path.startswith(_S3_LOCAL_MOUNT_PREFIX): - path = 's3://' + path[len(_S3_LOCAL_MOUNT_PREFIX):] - self.uri = path + self.uri = convert_local_path_to_remote_path(path) + + +def convert_local_path_to_remote_path(path: str) -> str: + if path.startswith(_GCS_LOCAL_MOUNT_PREFIX): + return 'gs://' + path[len(_GCS_LOCAL_MOUNT_PREFIX):] + elif path.startswith(_MINIO_LOCAL_MOUNT_PREFIX): + return 'minio://' + path[len(_MINIO_LOCAL_MOUNT_PREFIX):] + elif path.startswith(_S3_LOCAL_MOUNT_PREFIX): + return 's3://' + path[len(_S3_LOCAL_MOUNT_PREFIX):] + return path class Model(Artifact): @@ -112,12 +115,6 @@ class Model(Artifact): """ schema_title = 'system.Model' - def __init__(self, - name: Optional[str] = None, - uri: Optional[str] = None, - metadata: Optional[Dict] = None) -> None: - super().__init__(uri=uri, name=name, metadata=metadata) - @property def framework(self) -> str: return self._get_framework() @@ -143,12 +140,6 @@ class Dataset(Artifact): """ schema_title = 'system.Dataset' - def __init__(self, - name: Optional[str] = None, - uri: Optional[str] = None, - metadata: Optional[Dict] = None) -> None: - super().__init__(uri=uri, name=name, metadata=metadata) - class Metrics(Artifact): """An artifact for storing key-value scalar metrics. @@ -160,12 +151,6 @@ class Metrics(Artifact): """ schema_title = 'system.Metrics' - def __init__(self, - name: Optional[str] = None, - uri: Optional[str] = None, - metadata: Optional[Dict] = None) -> None: - super().__init__(uri=uri, name=name, metadata=metadata) - def log_metric(self, metric: str, value: float) -> None: """Sets a custom scalar metric in the artifact's metadata. @@ -186,12 +171,6 @@ class ClassificationMetrics(Artifact): """ schema_title = 'system.ClassificationMetrics' - def __init__(self, - name: Optional[str] = None, - uri: Optional[str] = None, - metadata: Optional[Dict] = None): - super().__init__(uri=uri, name=name, metadata=metadata) - def log_roc_data_point(self, fpr: float, tpr: float, threshold: float) -> None: """Logs a single data point in the ROC curve to metadata. @@ -355,12 +334,6 @@ class SlicedClassificationMetrics(Artifact): schema_title = 'system.SlicedClassificationMetrics' - def __init__(self, - name: Optional[str] = None, - uri: Optional[str] = None, - metadata: Optional[Dict] = None) -> None: - super().__init__(uri=uri, name=name, metadata=metadata) - def _upsert_classification_metrics_for_slice(self, slice: str) -> None: """Upserts the classification metrics instance for a slice.""" if slice not in self._sliced_metrics: @@ -479,12 +452,6 @@ class HTML(Artifact): """ schema_title = 'system.HTML' - def __init__(self, - name: Optional[str] = None, - uri: Optional[str] = None, - metadata: Optional[Dict] = None) -> None: - super().__init__(uri=uri, name=name, metadata=metadata) - class Markdown(Artifact): """An artifact representing a markdown file. @@ -496,12 +463,6 @@ class Markdown(Artifact): """ schema_title = 'system.Markdown' - def __init__(self, - name: Optional[str] = None, - uri: Optional[str] = None, - metadata: Optional[Dict] = None): - super().__init__(uri=uri, name=name, metadata=metadata) - _SCHEMA_TITLE_TO_TYPE: Dict[str, Type[Artifact]] = { x.schema_title: x for x in [ @@ -515,3 +476,38 @@ def __init__(self, Markdown, ] } + +CONTAINER_TASK_ROOT: Optional[str] = None + + +# suffix default of 'Output' should be the same key as the default key for a +# single output component, but use value not variable for reference docs +def get_uri(suffix: str = 'Output') -> str: + """Gets the task root URI, a unique object storage URI associated with the + current task. This function may only be called at task runtime. + + Returns an empty string if the task root cannot be inferred from the runtime environment. + + Args: + suffix: A suffix to append to the URI. This is a helpful for creating unique subdirectories when the component has multiple outputs. + + Returns: + The URI or empty string. + """ + if CONTAINER_TASK_ROOT is None: + raise RuntimeError( + f"'dsl.{get_uri.__name__}' can only be called at task runtime. The task root is unknown in the current environment." + ) + UNSUPPORTED_KFP_PATH = '/tmp/kfp_outputs' + if CONTAINER_TASK_ROOT == UNSUPPORTED_KFP_PATH: + warnings.warn( + f'dsl.{get_uri.__name__} is not yet supported by the KFP backend. Please specify a URI explicitly.', + RuntimeWarning, + stacklevel=2, + ) + # return empty string, not None, to conform with logic in artifact + # constructor which immediately converts uri=None to uri='' + # this way the .path property can worry about handling fewer input types + return '' + remote_task_root = convert_local_path_to_remote_path(CONTAINER_TASK_ROOT) + return os.path.join(remote_task_root, suffix) diff --git a/sdk/python/kfp/dsl/types/artifact_types_test.py b/sdk/python/kfp/dsl/types/artifact_types_test.py new file mode 100644 index 0000000000..c34f4a6bba --- /dev/null +++ b/sdk/python/kfp/dsl/types/artifact_types_test.py @@ -0,0 +1,150 @@ +# Copyright 2021 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for kfp.components.types.artifact_types.""" + +import contextlib +import json +import os +import unittest + +from absl.testing import parameterized +from kfp import dsl +from kfp.dsl.types import artifact_types + + +class ArtifactsTest(unittest.TestCase): + + def test_complex_metrics(self): + metrics = dsl.ClassificationMetrics() + metrics.log_roc_data_point(threshold=0.1, tpr=98.2, fpr=96.2) + metrics.log_roc_data_point(threshold=24.3, tpr=24.5, fpr=98.4) + metrics.set_confusion_matrix_categories(['dog', 'cat', 'horses']) + metrics.log_confusion_matrix_row('dog', [2, 6, 0]) + metrics.log_confusion_matrix_cell('cat', 'dog', 3) + metrics.log_confusion_matrix_cell('horses', 'horses', 3) + metrics.metadata['test'] = 1.0 + with open( + os.path.join( + os.path.dirname(__file__), 'test_data', + 'expected_io_types_classification_metrics.json') + ) as json_file: + expected_json = json.load(json_file) + self.assertEqual(expected_json, metrics.metadata) + + def test_complex_metrics_bulk_loading(self): + metrics = dsl.ClassificationMetrics() + metrics.log_roc_curve( + fpr=[85.1, 85.1, 85.1], + tpr=[52.6, 52.6, 52.6], + threshold=[53.6, 53.6, 53.6]) + metrics.log_confusion_matrix(['dog', 'cat', 'horses'], + [[2, 6, 0], [3, 5, 6], [5, 7, 8]]) + with open( + os.path.join( + os.path.dirname(__file__), 'test_data', + 'expected_io_types_bulk_load_classification_metrics.json') + ) as json_file: + expected_json = json.load(json_file) + self.assertEqual(expected_json, metrics.metadata) + + +@contextlib.contextmanager +def set_temporary_task_root(task_root: str): + artifact_types.CONTAINER_TASK_ROOT = task_root + try: + yield + finally: + artifact_types.CONTAINER_TASK_ROOT = None + + +class TestGetUri(unittest.TestCase): + + def test_raise_if_no_env_var(self): + + with self.assertRaisesRegex( + RuntimeError, + r"'dsl\.get_uri' can only be called at task runtime\. The task root is unknown in the current environment\." + ): + dsl.get_uri() + + def test_default_gcs(self): + with set_temporary_task_root( + '/gcs/my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789' + ): + self.assertEqual( + 'gs://my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789/Output', + dsl.get_uri()) + + def test_default_s3(self): + with set_temporary_task_root( + '/s3/my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789' + ): + self.assertEqual( + 's3://my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789/Output', + dsl.get_uri()) + + def test_default_minio(self): + with set_temporary_task_root( + '/minio/my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789' + ): + self.assertEqual( + 'minio://my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789/Output', + dsl.get_uri()) + + def test_suffix_arg_gcs(self): + with set_temporary_task_root( + '/gcs/my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789' + ): + self.assertEqual( + 'gs://my_bucket/123456789/abc-09-14-2023-14-21-53/foo_123456789/model', + dsl.get_uri('model')) + + def test_suffix_arg_tmp_no_suffix(self): + with set_temporary_task_root('/tmp/kfp_outputs'): + with self.assertWarnsRegex( + RuntimeWarning, + r'dsl\.get_uri is not yet supported by the KFP backend\. Please specify a URI explicitly\.' + ): + actual = dsl.get_uri('model') + self.assertEqual('', actual) + + def test_suffix_arg_tmp_with_suffix(self): + with set_temporary_task_root('/tmp/kfp_outputs'): + with self.assertWarnsRegex( + RuntimeWarning, + r'dsl\.get_uri is not yet supported by the KFP backend\. Please specify a URI explicitly\.' + ): + actual = dsl.get_uri('model') + self.assertEqual('', actual) + + +class TestConvertLocalPathToRemotePath(parameterized.TestCase): + + @parameterized.parameters([{ + 'local_path': local_path, + 'expected': expected + } for local_path, expected in [ + ('/gcs/foo/bar', 'gs://foo/bar'), + ('/minio/foo/bar', 'minio://foo/bar'), + ('/s3/foo/bar', 's3://foo/bar'), + ('/tmp/kfp_outputs', '/tmp/kfp_outputs'), + ('/some/random/path', '/some/random/path'), + ]]) + def test_gcs(self, local_path, expected): + actual = artifact_types.convert_local_path_to_remote_path(local_path) + self.assertEqual(actual, expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdk/python/kfp/components/types/custom_artifact_types.py b/sdk/python/kfp/dsl/types/custom_artifact_types.py similarity index 96% rename from sdk/python/kfp/components/types/custom_artifact_types.py rename to sdk/python/kfp/dsl/types/custom_artifact_types.py index 7174c86aa8..beba576dc6 100644 --- a/sdk/python/kfp/components/types/custom_artifact_types.py +++ b/sdk/python/kfp/dsl/types/custom_artifact_types.py @@ -16,9 +16,9 @@ import inspect from typing import Callable, Dict, List, Union -from kfp.components import component_factory -from kfp.components.types import type_annotations -from kfp.components.types import type_utils +from kfp.dsl import component_factory +from kfp.dsl.types import type_annotations +from kfp.dsl.types import type_utils RETURN_PREFIX = 'return-' @@ -53,9 +53,8 @@ def get_param_to_custom_artifact_class(func: Callable) -> Dict[str, type]: artifact_class = type_annotations.get_io_artifact_class(annotation) if artifact_class not in kfp_artifact_classes: param_to_artifact_cls[name] = artifact_class - elif type_annotations.is_artifact_class(annotation): - param_to_artifact_cls[name] = annotation - if artifact_class not in kfp_artifact_classes: + elif type_annotations.issubclass_of_artifact(annotation): + if annotation not in kfp_artifact_classes: param_to_artifact_cls[name] = artifact_class return_annotation = signature.return_annotation diff --git a/sdk/python/kfp/components/types/custom_artifact_types_test.py b/sdk/python/kfp/dsl/types/custom_artifact_types_test.py similarity index 96% rename from sdk/python/kfp/components/types/custom_artifact_types_test.py rename to sdk/python/kfp/dsl/types/custom_artifact_types_test.py index ed856db23b..4a9c279df6 100644 --- a/sdk/python/kfp/components/types/custom_artifact_types_test.py +++ b/sdk/python/kfp/dsl/types/custom_artifact_types_test.py @@ -24,14 +24,14 @@ from absl.testing import parameterized import kfp from kfp import dsl -from kfp.components.types import artifact_types -from kfp.components.types import custom_artifact_types -from kfp.components.types.artifact_types import Artifact -from kfp.components.types.artifact_types import Dataset -from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import OutputPath from kfp.dsl import Input from kfp.dsl import Output +from kfp.dsl.types import artifact_types +from kfp.dsl.types import custom_artifact_types +from kfp.dsl.types.artifact_types import Artifact +from kfp.dsl.types.artifact_types import Dataset +from kfp.dsl.types.type_annotations import InputPath +from kfp.dsl.types.type_annotations import OutputPath Alias = Artifact artifact_types_alias = artifact_types @@ -219,9 +219,9 @@ def func() -> typing.NamedTuple('Outputs', [ class TestGetFullQualnameForArtifact(_TestCaseWithThirdPartyPackage): # only gets called on artifacts, so don't need to test on all types @parameterized.parameters([ - (Alias, 'kfp.components.types.artifact_types.Artifact'), - (Artifact, 'kfp.components.types.artifact_types.Artifact'), - (Dataset, 'kfp.components.types.artifact_types.Dataset'), + (Alias, 'kfp.dsl.types.artifact_types.Artifact'), + (Artifact, 'kfp.dsl.types.artifact_types.Artifact'), + (Dataset, 'kfp.dsl.types.artifact_types.Dataset'), ]) def test(self, obj: Any, expected_qualname: str): self.assertEqual( diff --git a/sdk/python/kfp/components/types/test_data/expected_bulk_loaded_confusion_matrix.json b/sdk/python/kfp/dsl/types/test_data/expected_bulk_loaded_confusion_matrix.json similarity index 98% rename from sdk/python/kfp/components/types/test_data/expected_bulk_loaded_confusion_matrix.json rename to sdk/python/kfp/dsl/types/test_data/expected_bulk_loaded_confusion_matrix.json index 184233e0ba..1a994dff33 100644 --- a/sdk/python/kfp/components/types/test_data/expected_bulk_loaded_confusion_matrix.json +++ b/sdk/python/kfp/dsl/types/test_data/expected_bulk_loaded_confusion_matrix.json @@ -7,4 +7,4 @@ [2, 6, 0], [3, 5, 6], [5, 7, 8]] -} \ No newline at end of file +} diff --git a/sdk/python/kfp/components/types/test_data/expected_confusion_matrix.json b/sdk/python/kfp/dsl/types/test_data/expected_confusion_matrix.json similarity index 98% rename from sdk/python/kfp/components/types/test_data/expected_confusion_matrix.json rename to sdk/python/kfp/dsl/types/test_data/expected_confusion_matrix.json index 83312d1daa..560d6cbd16 100644 --- a/sdk/python/kfp/components/types/test_data/expected_confusion_matrix.json +++ b/sdk/python/kfp/dsl/types/test_data/expected_confusion_matrix.json @@ -7,4 +7,4 @@ [2, 6, 0], [3, 0, 0], [0, 0, 0]] -} \ No newline at end of file +} diff --git a/sdk/python/kfp/components/types/test_data/expected_io_types_bulk_load_classification_metrics.json b/sdk/python/kfp/dsl/types/test_data/expected_io_types_bulk_load_classification_metrics.json similarity index 83% rename from sdk/python/kfp/components/types/test_data/expected_io_types_bulk_load_classification_metrics.json rename to sdk/python/kfp/dsl/types/test_data/expected_io_types_bulk_load_classification_metrics.json index 356109ddcf..93092e04cd 100644 --- a/sdk/python/kfp/components/types/test_data/expected_io_types_bulk_load_classification_metrics.json +++ b/sdk/python/kfp/dsl/types/test_data/expected_io_types_bulk_load_classification_metrics.json @@ -1,11 +1,11 @@ { - "confidenceMetrics": + "confidenceMetrics": [ { "confidenceThreshold": 53.6, "recall": 52.6, "falsePositiveRate": 85.1 - }, + }, { "confidenceThreshold": 53.6, "recall": 52.6, @@ -16,20 +16,20 @@ "recall": 52.6, "falsePositiveRate": 85.1 } - ], - "confusionMatrix": + ], + "confusionMatrix": { - "annotationSpecs": + "annotationSpecs": [ {"displayName": "dog"}, {"displayName": "cat"}, {"displayName": "horses"} ], - "rows": + "rows": [ {"row": [2, 6, 0]}, {"row": [3, 5, 6]}, {"row" : [5, 7, 8]} ] } -} \ No newline at end of file +} diff --git a/sdk/python/kfp/components/types/test_data/expected_io_types_classification_metrics.json b/sdk/python/kfp/dsl/types/test_data/expected_io_types_classification_metrics.json similarity index 85% rename from sdk/python/kfp/components/types/test_data/expected_io_types_classification_metrics.json rename to sdk/python/kfp/dsl/types/test_data/expected_io_types_classification_metrics.json index 71324c8164..528cf611a7 100644 --- a/sdk/python/kfp/components/types/test_data/expected_io_types_classification_metrics.json +++ b/sdk/python/kfp/dsl/types/test_data/expected_io_types_classification_metrics.json @@ -5,22 +5,22 @@ "confidenceThreshold": 0.1, "recall": 98.2, "falsePositiveRate": 96.2 - }, + }, { "confidenceThreshold": 24.3, "recall": 24.5, "falsePositiveRate": 98.4 } ], - "confusionMatrix": + "confusionMatrix": { - "annotationSpecs": + "annotationSpecs": [ {"displayName": "dog"}, {"displayName": "cat"}, {"displayName": "horses"} - ], - "rows": + ], + "rows": [ {"row" : [2, 6, 0]}, {"row" : [3, 0, 0]}, diff --git a/sdk/python/kfp/components/types/type_annotations.py b/sdk/python/kfp/dsl/types/type_annotations.py similarity index 76% rename from sdk/python/kfp/components/types/type_annotations.py rename to sdk/python/kfp/dsl/types/type_annotations.py index 138a98198f..cd6adb89d8 100644 --- a/sdk/python/kfp/components/types/type_annotations.py +++ b/sdk/python/kfp/dsl/types/type_annotations.py @@ -17,11 +17,11 @@ """ import re -from typing import List, Type, TypeVar, Union +from typing import Any, List, Optional, Type, TypeVar, Union -from kfp.components.types import artifact_types -from kfp.components.types import type_annotations -from kfp.components.types import type_utils +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_annotations +from kfp.dsl.types import type_utils class OutputPath: @@ -99,7 +99,7 @@ def __eq__(self, other): def construct_type_for_inputpath_or_outputpath( - type_: Union[str, Type, None]) -> Union[str, None]: + type_: Union[str, Type, None]) -> Optional[str]: if type_annotations.is_artifact_class(type_): return type_utils.create_bundled_artifact_type(type_.schema_title, type_.schema_version) @@ -135,7 +135,7 @@ def is_Input_Output_artifact_annotation(typ) -> bool: return True -def is_input_artifact(typ) -> bool: +def is_artifact_wrapped_in_Input(typ: Any) -> bool: """Returns True if typ is of type Input[T].""" if not is_Input_Output_artifact_annotation(typ): return False @@ -143,7 +143,7 @@ def is_input_artifact(typ) -> bool: return typ.__metadata__[0] == InputAnnotation -def is_output_artifact(typ) -> bool: +def is_artifact_wrapped_in_Output(typ: Any) -> bool: """Returns True if typ is of type Output[T].""" if not is_Input_Output_artifact_annotation(typ): return False @@ -160,14 +160,19 @@ def get_io_artifact_class(typ): return None # extract inner type from list of artifacts - inner = typ.__args__[0] + inner = strip_Input_or_Output_marker(typ) if hasattr(inner, '__origin__') and inner.__origin__ == list: return inner.__args__[0] return inner -def get_io_artifact_annotation(typ): +def strip_Input_or_Output_marker(typ: Any) -> artifact_types.Artifact: + return typ.__args__[0] + + +def get_input_or_output_marker( + typ) -> Optional[Union[InputAnnotation, OutputAnnotation]]: if not is_Input_Output_artifact_annotation(typ): return None @@ -233,13 +238,40 @@ def is_artifact_class(artifact_class_or_instance: Type) -> bool: def is_list_of_artifacts( - type_var: Union[Type[List[artifact_types.Artifact]], - Type[artifact_types.Artifact]] + annotation: Union[Type[List[artifact_types.Artifact]], + Type[artifact_types.Artifact]] ) -> bool: - # the type annotation for this function's `type_var` parameter may not actually be a subclass of the KFP SDK's Artifact class for custom artifact types - is_list_or_list_generic = getattr(type_var, '__origin__', None) == list - # in >= python3.9, List wont have .__args__ if it's used as `-> List` with no inner type argument - contains_artifact = hasattr( - type_var, '__args__') and type_annotations.is_artifact_class( - type_var.__args__[0]) - return is_list_or_list_generic and contains_artifact + """Checks if an object is a list of list of artifacts annotation (e.g., + List[Artifact], List[Dataset])""" + return is_generic_list(annotation) and issubclass_of_artifact( + get_inner_type(annotation)) + + +def get_inner_type(annotation: Any) -> Optional[Any]: + """Returns the inner type of a generic annotation. + + For Union or Optional types with multiple inner types, a tuple of + types is returned. + """ + # Check if the annotation has '__args__' attribute + if hasattr(annotation, '__args__'): + if len(annotation.__args__) == 1: + return annotation.__args__[0] + else: + return tuple(annotation.__args__) + return None + + +def issubclass_of_artifact(obj: Any) -> bool: + """Checks if an object is a class and a subclass of a dsl.Artifact.""" + return type(obj) == type and issubclass(obj, artifact_types.Artifact) + + +def is_generic_list(annotation: Any) -> bool: + # handles generics from the typing module for python<3.9 + typing_generic_list = getattr(annotation, '__origin__', + None) is list or getattr( + annotation, '__origin__', None) is List + # handles built-in generics for python>=3.9 + built_in_generic_list = annotation == list + return typing_generic_list or built_in_generic_list diff --git a/sdk/python/kfp/components/types/type_annotations_test.py b/sdk/python/kfp/dsl/types/type_annotations_test.py similarity index 62% rename from sdk/python/kfp/components/types/type_annotations_test.py rename to sdk/python/kfp/dsl/types/type_annotations_test.py index f514cdd2fa..df34682ce0 100644 --- a/sdk/python/kfp/components/types/type_annotations_test.py +++ b/sdk/python/kfp/dsl/types/type_annotations_test.py @@ -11,21 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for kfp.components.types.type_annotations.""" +"""Tests for kfp.dsl.types.type_annotations.""" -from typing import Any, Dict, List, Optional +import sys +from typing import Any, Dict, List, Optional, Union import unittest from absl.testing import parameterized -from kfp.components.types import artifact_types -from kfp.components.types import type_annotations -from kfp.components.types.artifact_types import Model -from kfp.components.types.type_annotations import InputAnnotation -from kfp.components.types.type_annotations import InputPath -from kfp.components.types.type_annotations import OutputAnnotation -from kfp.components.types.type_annotations import OutputPath +from kfp import dsl from kfp.dsl import Input from kfp.dsl import Output +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_annotations +from kfp.dsl.types.artifact_types import Model +from kfp.dsl.types.type_annotations import InputAnnotation +from kfp.dsl.types.type_annotations import InputPath +from kfp.dsl.types.type_annotations import OutputAnnotation +from kfp.dsl.types.type_annotations import OutputPath class AnnotationsTest(parameterized.TestCase): @@ -56,21 +58,24 @@ def test_is_not_artifact_annotation(self, annotation): Input, ]) def test_is_input_artifact(self, annotation): - self.assertTrue(type_annotations.is_input_artifact(annotation)) + self.assertTrue( + type_annotations.is_artifact_wrapped_in_Input(annotation)) @parameterized.parameters([ Output[Model], Output, ]) def test_is_not_input_artifact(self, annotation): - self.assertFalse(type_annotations.is_input_artifact(annotation)) + self.assertFalse( + type_annotations.is_artifact_wrapped_in_Input(annotation)) @parameterized.parameters([ Output[Model], Output[List[Model]], ]) def test_is_output_artifact(self, annotation): - self.assertTrue(type_annotations.is_output_artifact(annotation)) + self.assertTrue( + type_annotations.is_artifact_wrapped_in_Output(annotation)) @parameterized.parameters([ Input[Model], @@ -78,7 +83,8 @@ def test_is_output_artifact(self, annotation): Input, ]) def test_is_not_output_artifact(self, annotation): - self.assertFalse(type_annotations.is_output_artifact(annotation)) + self.assertFalse( + type_annotations.is_artifact_wrapped_in_Output(annotation)) def test_get_io_artifact_class(self): self.assertEqual( @@ -95,26 +101,26 @@ def test_get_io_artifact_class(self): def test_get_io_artifact_annotation(self): self.assertEqual( - type_annotations.get_io_artifact_annotation(Output[Model]), + type_annotations.get_input_or_output_marker(Output[Model]), OutputAnnotation) self.assertEqual( - type_annotations.get_io_artifact_annotation(Output[List[Model]]), + type_annotations.get_input_or_output_marker(Output[List[Model]]), OutputAnnotation) self.assertEqual( - type_annotations.get_io_artifact_annotation(Input[Model]), + type_annotations.get_input_or_output_marker(Input[Model]), InputAnnotation) self.assertEqual( - type_annotations.get_io_artifact_annotation(Input[List[Model]]), + type_annotations.get_input_or_output_marker(Input[List[Model]]), InputAnnotation) self.assertEqual( - type_annotations.get_io_artifact_annotation(Input), InputAnnotation) + type_annotations.get_input_or_output_marker(Input), InputAnnotation) self.assertEqual( - type_annotations.get_io_artifact_annotation(Output), + type_annotations.get_input_or_output_marker(Output), OutputAnnotation) self.assertEqual( - type_annotations.get_io_artifact_annotation(Model), None) - self.assertEqual(type_annotations.get_io_artifact_annotation(str), None) + type_annotations.get_input_or_output_marker(Model), None) + self.assertEqual(type_annotations.get_input_or_output_marker(str), None) @parameterized.parameters( { @@ -209,17 +215,102 @@ def test_false(self, obj): def test_false_no_schema_title(self): - class NotArtifact: + class MissingSchemaTitle: schema_version = '' - self.assertFalse(type_annotations.is_artifact_class(NotArtifact)) + self.assertFalse(type_annotations.is_artifact_class(MissingSchemaTitle)) def test_false_no_schema_version(self): - class NotArtifact: + class MissingSchemaVersion: schema_title = '' - self.assertFalse(type_annotations.is_artifact_class(NotArtifact)) + self.assertFalse( + type_annotations.is_artifact_class(MissingSchemaVersion)) + + +class ArtifactSubclass(dsl.Artifact): + pass + + +class NotArtifactSubclass: + pass + + +class TestIsSubclassOfArtifact(parameterized.TestCase): + + @parameterized.parameters([{ + 'obj': obj + } for obj in [ + dsl.Artifact, + dsl.Dataset, + dsl.Metrics, + ArtifactSubclass, + ]]) + def test_true(self, obj): + self.assertTrue(type_annotations.issubclass_of_artifact(obj)) + + @parameterized.parameters([{ + 'obj': obj + } for obj in [ + dsl.Artifact(), + dsl.Dataset(), + 1, + NotArtifactSubclass, + ]]) + def test_false(self, obj): + self.assertFalse(type_annotations.issubclass_of_artifact(obj)) + + +class TestIsGenericList(parameterized.TestCase): + + @parameterized.parameters([{ + 'obj': obj + } for obj in [ + List, + List[str], + List[dsl.Artifact], + List[Dict[str, str]], + ] + ([ + list, + list[str], + ] if sys.version_info >= (3, 9, 0) else [])]) + def test_true(self, obj): + self.assertTrue(type_annotations.is_generic_list(obj)) + + @parameterized.parameters([{ + 'obj': obj + } for obj in [ + Optional[List[str]], + Dict[str, str], + str, + int, + dsl.Artifact, + ]]) + def test_false(self, obj): + self.assertFalse(type_annotations.is_generic_list(obj)) + + +class TestGetInnerType(parameterized.TestCase): + + @parameterized.parameters([{ + 'annotation': annotation, + 'expected': expected + } for annotation, expected in [ + (int, None), + (Optional[int], (int, type(None))), + (Union[int, None], (int, type(None))), + (List[str], str), + (Dict[str, str], (str, str)), + (List[dsl.Artifact], dsl.Artifact), + ] + ([ + (list[str], str), + (dict[str, str], (str, str)), + (list[dsl.Artifact], dsl.Artifact), + ] if sys.version_info >= (3, 9, 0) else [])]) + def test(self, annotation, expected): + actual = type_annotations.get_inner_type(annotation) + self.assertEqual(actual, expected) if __name__ == '__main__': diff --git a/sdk/python/kfp/components/types/type_utils.py b/sdk/python/kfp/dsl/types/type_utils.py similarity index 86% rename from sdk/python/kfp/components/types/type_utils.py rename to sdk/python/kfp/dsl/types/type_utils.py index 92b9328fab..cd84a37041 100644 --- a/sdk/python/kfp/components/types/type_utils.py +++ b/sdk/python/kfp/dsl/types/type_utils.py @@ -20,12 +20,9 @@ import warnings import kfp -from kfp.components import pipeline_channel -from kfp.components import structures -from kfp.components import task_final_status -from kfp.components.types import artifact_types -from kfp.components.types import type_annotations -from kfp.pipeline_spec import pipeline_spec_pb2 +from kfp.dsl import task_final_status +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_annotations DEFAULT_ARTIFACT_SCHEMA_VERSION = '0.0.1' PARAMETER_TYPES = Union[str, int, float, bool, dict, list] @@ -48,20 +45,29 @@ # ComponentSpec I/O types to (IR) PipelineTaskSpec I/O types mapping. # The keys are normalized (lowercased). These are types viewed as Parameters. # The values are the corresponding IR parameter primitive types. -_PARAMETER_TYPES_MAPPING = { - 'integer': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER, - 'int': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER, - 'double': pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE, - 'float': pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE, - 'string': pipeline_spec_pb2.ParameterType.STRING, - 'str': pipeline_spec_pb2.ParameterType.STRING, - 'text': pipeline_spec_pb2.ParameterType.STRING, - 'bool': pipeline_spec_pb2.ParameterType.BOOLEAN, - 'boolean': pipeline_spec_pb2.ParameterType.BOOLEAN, - 'dict': pipeline_spec_pb2.ParameterType.STRUCT, - 'list': pipeline_spec_pb2.ParameterType.LIST, - 'jsonobject': pipeline_spec_pb2.ParameterType.STRUCT, - 'jsonarray': pipeline_spec_pb2.ParameterType.LIST, + +# pipeline_spec_pb2.ParameterType enum values +NUMBER_DOUBLE = 1 +NUMBER_INTEGER = 2 +STRING = 3 +BOOLEAN = 4 +LIST = 5 +STRUCT = 6 +TASK_FINAL_STATUS = 7 +PARAMETER_TYPES_MAPPING = { + 'integer': NUMBER_INTEGER, + 'int': NUMBER_INTEGER, + 'double': NUMBER_DOUBLE, + 'float': NUMBER_DOUBLE, + 'string': STRING, + 'str': STRING, + 'text': STRING, + 'bool': BOOLEAN, + 'boolean': BOOLEAN, + 'dict': STRUCT, + 'list': LIST, + 'jsonobject': STRUCT, + 'jsonarray': LIST, } @@ -110,18 +116,6 @@ def deserialize_v1_component_yaml_default(type_: str, default: Any) -> Any: return default -# Mapping primitive types to their IR message field names. -# This is used in constructing condition strings. -_PARAMETER_TYPES_VALUE_REFERENCE_MAPPING = { - pipeline_spec_pb2.ParameterType.NUMBER_INTEGER: 'number_value', - pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE: 'number_value', - pipeline_spec_pb2.ParameterType.STRING: 'string_value', - pipeline_spec_pb2.ParameterType.BOOLEAN: 'bool_value', - pipeline_spec_pb2.ParameterType.STRUCT: 'struct_value', - pipeline_spec_pb2.ParameterType.LIST: 'list_value', -} - - def is_task_final_status_type(type_name: Optional[Union[str, dict]]) -> bool: """Check if a ComponentSpec I/O type is PipelineTaskFinalStatus. @@ -152,14 +146,17 @@ def is_parameter_type(type_name: Optional[Union[str, dict]]) -> bool: return False return type_name.lower( - ) in _PARAMETER_TYPES_MAPPING or is_task_final_status_type(type_name) + ) in PARAMETER_TYPES_MAPPING or is_task_final_status_type(type_name) def bundled_artifact_to_artifact_proto( - bundled_artifact_str: str) -> pipeline_spec_pb2.ArtifactTypeSchema: + bundled_artifact_str: str) -> 'pipeline_spec_pb2.ArtifactTypeSchema': """Gets the IR ArtifactTypeSchema proto for a bundled artifact in form `.@x.x.x` (e.g., system.Artifact@0.0.1).""" bundled_artifact_str, schema_version = bundled_artifact_str.split('@') + + from kfp.pipeline_spec import pipeline_spec_pb2 + return pipeline_spec_pb2.ArtifactTypeSchema( schema_title=bundled_artifact_str, schema_version=schema_version, @@ -168,7 +165,7 @@ def bundled_artifact_to_artifact_proto( def get_parameter_type( param_type: Optional[Union[Type, str, dict]] -) -> pipeline_spec_pb2.ParameterType: +) -> 'pipeline_spec_pb2.ParameterType': """Get the IR I/O parameter type for the given ComponentSpec I/O type. Args: @@ -190,32 +187,21 @@ def get_parameter_type( type_name = list(param_type.keys())[0] else: type_name = type_annotations.get_short_type_name(str(param_type)) - return _PARAMETER_TYPES_MAPPING.get(type_name.lower()) + return PARAMETER_TYPES_MAPPING.get(type_name.lower()) def get_parameter_type_name( param_type: Optional[Union[Type, str, dict]]) -> str: """Gets the parameter type name.""" - return pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( - get_parameter_type(param_type)) - - -def get_parameter_type_field_name(type_name: Optional[str]) -> Optional[str]: - """Get the IR field name for the given primitive type. - - For example: 'str' -> 'string_value', 'double' -> 'double_value', etc. - Args: - type_name: type name of the ComponentSpec I/O primitive type. - - Returns: - The IR value reference field name. - - Raises: - AttributeError: if type_name is not a string type. - """ - return _PARAMETER_TYPES_VALUE_REFERENCE_MAPPING.get( - get_parameter_type(type_name)) + from kfp.pipeline_spec import pipeline_spec_pb2 + param_enum_val = get_parameter_type(param_type) + if param_enum_val is None: + raise ValueError( + '`param_type` is not a parameter type. Cannot get ParameterType name.' + ) + return pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( + param_enum_val) class InconsistentTypeException(Exception): @@ -231,9 +217,9 @@ def _get_type_string_from_component_argument( argument_value: Union['pipeline_channel.PipelineChannel', str, bool, int, float, dict, list] ) -> str: - # argument is a PipelineChannel - if isinstance(argument_value, - kfp.components.pipeline_channel.PipelineChannel): + # avoid circular imports + from kfp.dsl import pipeline_channel + if isinstance(argument_value, pipeline_channel.PipelineChannel): return argument_value.channel_type # argument is a constant @@ -249,9 +235,10 @@ def _get_type_string_from_component_argument( def verify_type_compatibility( given_value: Union['pipeline_channel.PipelineChannel', str, bool, int, float, dict, list], - expected_spec: Union[structures.InputSpec, structures.OutputSpec], + expected_spec: Union['structures.InputSpec', 'structures.OutputSpec'], error_message_prefix: str, checks_input: bool = True, + raise_on_error: bool = True, ) -> bool: """Verifies the given argument type is compatible with the expected type. @@ -260,12 +247,13 @@ def verify_type_compatibility( expected_spec: The InputSpec or OutputSpec that describes the expected type of given_value. error_message_prefix: The prefix for the error message. checks_input: True if checks an argument (given_value) against a component/pipeline input type (expected_spec). False if checks a component output (argument_value) against the pipeline output type (expected_spec). + raise_on_error: Whether to raise on type compatibility error. Should be passed kfp.TYPE_CHECK. Returns: True if types are compatible, and False if otherwise. Raises: - InconsistentTypeException if types are incompatible and TYPE_CHECK==True. + InconsistentTypeException if raise_on_error=True. """ # extract and normalize types expected_type = expected_spec.type @@ -308,7 +296,7 @@ def verify_type_compatibility( else: error_message_suffix = f'Output of type {given_type!r} cannot be surfaced as pipeline output type {expected_type!r}' error_text = error_message_prefix + error_message_suffix - if kfp.TYPE_CHECK: + if raise_on_error: raise InconsistentTypeException(error_text) else: warnings.warn(InconsistentTypeWarning(error_text)) diff --git a/sdk/python/kfp/components/types/type_utils_test.py b/sdk/python/kfp/dsl/types/type_utils_test.py similarity index 98% rename from sdk/python/kfp/components/types/type_utils_test.py rename to sdk/python/kfp/dsl/types/type_utils_test.py index 6c36483b11..ee2cf16180 100644 --- a/sdk/python/kfp/components/types/type_utils_test.py +++ b/sdk/python/kfp/dsl/types/type_utils_test.py @@ -21,16 +21,16 @@ from kfp import compiler from kfp import components from kfp import dsl -from kfp.components import base_component -from kfp.components import pipeline_channel -from kfp.components import structures -from kfp.components import yaml_component -from kfp.components.types import artifact_types -from kfp.components.types import type_utils -from kfp.components.types.type_utils import InconsistentTypeException +from kfp.dsl import base_component from kfp.dsl import Dataset from kfp.dsl import Input from kfp.dsl import Output +from kfp.dsl import pipeline_channel +from kfp.dsl import structures +from kfp.dsl import yaml_component +from kfp.dsl.types import artifact_types +from kfp.dsl.types import type_utils +from kfp.dsl.types.type_utils import InconsistentTypeException from kfp.pipeline_spec import pipeline_spec_pb2 as pb _PARAMETER_TYPES = [ diff --git a/sdk/python/kfp/components/utils.py b/sdk/python/kfp/dsl/utils.py similarity index 100% rename from sdk/python/kfp/components/utils.py rename to sdk/python/kfp/dsl/utils.py diff --git a/sdk/python/kfp/components/utils_test.py b/sdk/python/kfp/dsl/utils_test.py similarity index 98% rename from sdk/python/kfp/components/utils_test.py rename to sdk/python/kfp/dsl/utils_test.py index 1a3df52f07..a0be587fc0 100644 --- a/sdk/python/kfp/components/utils_test.py +++ b/sdk/python/kfp/dsl/utils_test.py @@ -11,12 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for kfp.components.utils.""" +"""Tests for kfp.dsl.utils.""" import unittest from absl.testing import parameterized -from kfp.components import utils +from kfp.dsl import utils class UtilsTest(parameterized.TestCase): diff --git a/sdk/python/kfp/components/v1_modelbase.py b/sdk/python/kfp/dsl/v1_modelbase.py similarity index 100% rename from sdk/python/kfp/components/v1_modelbase.py rename to sdk/python/kfp/dsl/v1_modelbase.py diff --git a/sdk/python/kfp/components/v1_structures.py b/sdk/python/kfp/dsl/v1_structures.py similarity index 98% rename from sdk/python/kfp/components/v1_structures.py rename to sdk/python/kfp/dsl/v1_structures.py index 90818f260c..57cc7c6375 100644 --- a/sdk/python/kfp/components/v1_structures.py +++ b/sdk/python/kfp/dsl/v1_structures.py @@ -15,9 +15,7 @@ from collections import OrderedDict from typing import Any, Dict, List, Mapping, Optional, Union -import yaml - -from .v1_modelbase import ModelBase +from kfp.dsl.v1_modelbase import ModelBase PrimitiveTypes = Union[str, int, float, bool] PrimitiveTypesIncludingNone = Optional[PrimitiveTypes] @@ -438,17 +436,6 @@ def verify_arg(arg): f'Argument "{argument}" references non-existing input.' ) - def save(self, file_path: str): - """Saves the component definition to file. - - It can be shared online and later loaded using the - load_component function. - """ - - component_yaml = yaml.dump(self.to_dict(), sort_keys=True) - with open(file_path, 'w') as f: - f.write(component_yaml) - class ComponentReference(ModelBase): """Component reference. diff --git a/sdk/python/kfp/dsl/yaml_component.py b/sdk/python/kfp/dsl/yaml_component.py new file mode 100644 index 0000000000..187fa533f2 --- /dev/null +++ b/sdk/python/kfp/dsl/yaml_component.py @@ -0,0 +1,54 @@ +# Copyright 2021-2022 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Component loaded from YAML.""" + +from google.protobuf import json_format +from kfp.dsl import base_component +from kfp.dsl import structures +from kfp.pipeline_spec import pipeline_spec_pb2 + + +class YamlComponent(base_component.BaseComponent): + """A component loaded from a YAML file. + + **Note:** ``YamlComponent`` is not intended to be used to construct components directly. Use ``kfp.components.load_component_from_*()`` instead. + + Attribute: + component_spec: Component definition. + component_yaml: The yaml string that this component is loaded from. + """ + + def __init__( + self, + component_spec: structures.ComponentSpec, + component_yaml: str, + ): + super().__init__(component_spec=component_spec) + self.component_yaml = component_yaml + + @property + def pipeline_spec(self) -> pipeline_spec_pb2.PipelineSpec: + """Returns the pipeline spec of the component.""" + component_dict = structures.load_documents_from_yaml( + self.component_yaml)[0] + is_v1 = 'implementation' in set(component_dict.keys()) + if is_v1: + return self.component_spec.to_pipeline_spec() + else: + return json_format.ParseDict(component_dict, + pipeline_spec_pb2.PipelineSpec()) + + def execute(self, *args, **kwargs): + """Not implemented.""" + raise NotImplementedError diff --git a/sdk/python/kfp/init_test.py b/sdk/python/kfp/init_test.py new file mode 100644 index 0000000000..9e6a86598a --- /dev/null +++ b/sdk/python/kfp/init_test.py @@ -0,0 +1,36 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import sys +import unittest +from unittest import mock + + +@mock.patch.object(sys, 'version_info', new=(3, 7, 12, 'final', 0)) +class TestPythonEOLWarning(unittest.TestCase): + + def test(self): + mod = importlib.import_module('kfp') + + with self.assertWarnsRegex( + FutureWarning, + r'Python 3\.7 has reached end-of-life\. KFP will drop support for Python 3\.7 on April 23, 2024\. To use new versions of the KFP SDK after that date, you will need to upgrade to Python >= 3\.8\. See https:\/\/devguide\.python\.org\/versions\/ for more details\.' + ): + # simulate first import from kfp + importlib.reload(mod) + + +if __name__ == '__main__': + unittest.main() diff --git a/sdk/python/kfp/v2/__init__.py b/sdk/python/kfp/v2/__init__.py index 82c9014ab9..fe854231ae 100644 --- a/sdk/python/kfp/v2/__init__.py +++ b/sdk/python/kfp/v2/__init__.py @@ -21,6 +21,6 @@ category=DeprecationWarning, stacklevel=2) -from kfp import compiler -from kfp import components -from kfp import dsl +from kfp import compiler # noqa: keep unused import +from kfp import components # noqa: keep unused import +from kfp import dsl # noqa: keep unused import diff --git a/sdk/python/test_data/components/add_numbers.yaml b/sdk/python/test_data/components/add_numbers.yaml index 33654302b6..bee5dff8f1 100644 --- a/sdk/python/test_data/components/add_numbers.yaml +++ b/sdk/python/test_data/components/add_numbers.yaml @@ -32,15 +32,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -81,4 +83,4 @@ root: Output: parameterType: NUMBER_INTEGER schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/component_with_metadata_fields.yaml b/sdk/python/test_data/components/component_with_metadata_fields.yaml index 16bdc539b1..0429b3fe7f 100644 --- a/sdk/python/test_data/components/component_with_metadata_fields.yaml +++ b/sdk/python/test_data/components/component_with_metadata_fields.yaml @@ -48,15 +48,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -124,4 +126,4 @@ root: description: The concatenated string. parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/component_with_pip_install.yaml b/sdk/python/test_data/components/component_with_pip_install.yaml index d9b8cf7a13..d1ae009cfa 100644 --- a/sdk/python/test_data/components/component_with_pip_install.yaml +++ b/sdk/python/test_data/components/component_with_pip_install.yaml @@ -17,16 +17,19 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location --index-url\ - \ https://pypi.org/simple --trusted-host https://pypi.org/simple 'yapf'\ - \ 'kfp==2.0.0-beta.16' && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple\ + \ --trusted-host https://pypi.org/simple 'kfp==2.1.3' '--no-deps' 'typing-extensions>=3.7.4,<5;\ + \ python_version<\"3.9\"' && python3 -m pip install --quiet --no-warn-script-location\ + \ --index-url https://pypi.org/simple --trusted-host https://pypi.org/simple\ + \ 'yapf' && \"$0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -46,4 +49,4 @@ root: taskInfo: name: component-with-pip-install schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/component_with_task_final_status.yaml b/sdk/python/test_data/components/component_with_task_final_status.yaml index e0a4bda25d..b88209fca6 100644 --- a/sdk/python/test_data/components/component_with_task_final_status.yaml +++ b/sdk/python/test_data/components/component_with_task_final_status.yaml @@ -24,15 +24,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -61,4 +63,4 @@ root: isOptional: true parameterType: TASK_FINAL_STATUS schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.17 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/concat_message.yaml b/sdk/python/test_data/components/concat_message.yaml index 4f0e97a25d..2f518f3ce7 100644 --- a/sdk/python/test_data/components/concat_message.yaml +++ b/sdk/python/test_data/components/concat_message.yaml @@ -32,15 +32,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -82,4 +84,4 @@ root: Output: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/container_io.yaml b/sdk/python/test_data/components/container_io.yaml index 19812e3046..e575a3fa30 100644 --- a/sdk/python/test_data/components/container_io.yaml +++ b/sdk/python/test_data/components/container_io.yaml @@ -57,4 +57,4 @@ root: output_path: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/container_no_input.yaml b/sdk/python/test_data/components/container_no_input.yaml index a9f3bcad81..cc373a5599 100644 --- a/sdk/python/test_data/components/container_no_input.yaml +++ b/sdk/python/test_data/components/container_no_input.yaml @@ -24,4 +24,4 @@ root: taskInfo: name: container-no-input schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/container_with_artifact_output.yaml b/sdk/python/test_data/components/container_with_artifact_output.yaml index ba35de0d70..baf5d5e257 100644 --- a/sdk/python/test_data/components/container_with_artifact_output.yaml +++ b/sdk/python/test_data/components/container_with_artifact_output.yaml @@ -79,4 +79,4 @@ root: model_config_path: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/container_with_concat_placeholder.yaml b/sdk/python/test_data/components/container_with_concat_placeholder.yaml index f234d6477b..95d486495a 100644 --- a/sdk/python/test_data/components/container_with_concat_placeholder.yaml +++ b/sdk/python/test_data/components/container_with_concat_placeholder.yaml @@ -73,4 +73,4 @@ root: output_path: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/container_with_if_placeholder.yaml b/sdk/python/test_data/components/container_with_if_placeholder.yaml index 165fafcde0..e81b17aeda 100644 --- a/sdk/python/test_data/components/container_with_if_placeholder.yaml +++ b/sdk/python/test_data/components/container_with_if_placeholder.yaml @@ -81,4 +81,4 @@ root: output_path: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/container_with_placeholder_in_fstring.yaml b/sdk/python/test_data/components/container_with_placeholder_in_fstring.yaml index 2dde86e657..cdf6fc7b1f 100644 --- a/sdk/python/test_data/components/container_with_placeholder_in_fstring.yaml +++ b/sdk/python/test_data/components/container_with_placeholder_in_fstring.yaml @@ -63,4 +63,4 @@ root: schemaTitle: system.Artifact schemaVersion: 0.0.1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/containerized_python_component.py b/sdk/python/test_data/components/containerized_python_component.py new file mode 100644 index 0000000000..041722d97f --- /dev/null +++ b/sdk/python/test_data/components/containerized_python_component.py @@ -0,0 +1,26 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from kfp import dsl + + +@dsl.component(base_image='python:3.7', target_image='kfp-image') +def concat_message(message1: str, message2: str) -> str: + return message1 + message2 + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=concat_message, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/components/containerized_python_component.yaml b/sdk/python/test_data/components/containerized_python_component.yaml new file mode 100644 index 0000000000..2e3423e2d7 --- /dev/null +++ b/sdk/python/test_data/components/containerized_python_component.yaml @@ -0,0 +1,70 @@ +# PIPELINE DEFINITION +# Name: concat-message +# Inputs: +# message1: str +# message2: str +# Outputs: +# Output: str +components: + comp-concat-message: + executorLabel: exec-concat-message + inputDefinitions: + parameters: + message1: + parameterType: STRING + message2: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING +deploymentSpec: + executors: + exec-concat-message: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - concat_message + command: + - python3 + - -m + - kfp.dsl.executor_main + image: kfp-image +pipelineInfo: + name: concat-message +root: + dag: + outputs: + parameters: + Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: concat-message + tasks: + concat-message: + cachingOptions: + enableCache: true + componentRef: + name: comp-concat-message + inputs: + parameters: + message1: + componentInputParameter: message1 + message2: + componentInputParameter: message2 + taskInfo: + name: concat-message + inputDefinitions: + parameters: + message1: + parameterType: STRING + message2: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/dict_input.yaml b/sdk/python/test_data/components/dict_input.yaml index 1eeac295b0..d0b73f3b50 100644 --- a/sdk/python/test_data/components/dict_input.yaml +++ b/sdk/python/test_data/components/dict_input.yaml @@ -23,15 +23,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -58,4 +60,4 @@ root: struct: parameterType: STRUCT schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/identity.yaml b/sdk/python/test_data/components/identity.yaml index 17b6b9afef..97fc9c3835 100644 --- a/sdk/python/test_data/components/identity.yaml +++ b/sdk/python/test_data/components/identity.yaml @@ -29,15 +29,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -74,4 +76,4 @@ root: Output: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/input_artifact.yaml b/sdk/python/test_data/components/input_artifact.yaml index 813ecc83e9..3aef4ef579 100644 --- a/sdk/python/test_data/components/input_artifact.yaml +++ b/sdk/python/test_data/components/input_artifact.yaml @@ -25,15 +25,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -63,4 +65,4 @@ root: schemaTitle: system.Dataset schemaVersion: 0.0.1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/nested_return.yaml b/sdk/python/test_data/components/nested_return.yaml index e8224c18da..337e20c6ba 100644 --- a/sdk/python/test_data/components/nested_return.yaml +++ b/sdk/python/test_data/components/nested_return.yaml @@ -23,15 +23,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -61,4 +63,4 @@ root: Output: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/output_metrics.yaml b/sdk/python/test_data/components/output_metrics.yaml index 3b67e606fc..abc3e39fdb 100644 --- a/sdk/python/test_data/components/output_metrics.yaml +++ b/sdk/python/test_data/components/output_metrics.yaml @@ -27,15 +27,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -77,4 +79,4 @@ root: schemaTitle: system.Metrics schemaVersion: 0.0.1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/components/preprocess.yaml b/sdk/python/test_data/components/preprocess.yaml index 05fff3f4cf..003cfec408 100644 --- a/sdk/python/test_data/components/preprocess.yaml +++ b/sdk/python/test_data/components/preprocess.yaml @@ -56,15 +56,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -171,4 +173,4 @@ root: output_parameter_path: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/component_with_optional_inputs.yaml b/sdk/python/test_data/pipelines/component_with_optional_inputs.yaml index 05ad4c2d2a..bbf7062cf1 100644 --- a/sdk/python/test_data/pipelines/component_with_optional_inputs.yaml +++ b/sdk/python/test_data/pipelines/component_with_optional_inputs.yaml @@ -29,15 +29,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -68,4 +70,4 @@ root: taskInfo: name: component-op schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/component_with_pip_index_urls.yaml b/sdk/python/test_data/pipelines/component_with_pip_index_urls.yaml index ddb43d85d3..9b0afa75ac 100644 --- a/sdk/python/test_data/pipelines/component_with_pip_index_urls.yaml +++ b/sdk/python/test_data/pipelines/component_with_pip_index_urls.yaml @@ -17,16 +17,19 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location --index-url\ - \ https://pypi.org/simple --trusted-host https://pypi.org/simple 'yapf'\ - \ 'kfp==2.0.0-beta.16' && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple\ + \ --trusted-host https://pypi.org/simple 'kfp==2.1.3' '--no-deps' 'typing-extensions>=3.7.4,<5;\ + \ python_version<\"3.9\"' && python3 -m pip install --quiet --no-warn-script-location\ + \ --index-url https://pypi.org/simple --trusted-host https://pypi.org/simple\ + \ 'yapf' && \"$0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -45,4 +48,4 @@ root: taskInfo: name: component-op schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/components_with_optional_artifacts.yaml b/sdk/python/test_data/pipelines/components_with_optional_artifacts.yaml index 25f4a0767e..3b034d8542 100644 --- a/sdk/python/test_data/pipelines/components_with_optional_artifacts.yaml +++ b/sdk/python/test_data/pipelines/components_with_optional_artifacts.yaml @@ -126,15 +126,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -155,15 +157,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -237,4 +241,4 @@ root: schemaVersion: 0.0.1 isOptional: true schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/container_component_with_no_inputs.yaml b/sdk/python/test_data/pipelines/container_component_with_no_inputs.yaml index c27d712d8c..01daa9fe71 100644 --- a/sdk/python/test_data/pipelines/container_component_with_no_inputs.yaml +++ b/sdk/python/test_data/pipelines/container_component_with_no_inputs.yaml @@ -24,4 +24,4 @@ root: taskInfo: name: hello-world-container schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/if_elif_else_complex.py b/sdk/python/test_data/pipelines/if_elif_else_complex.py new file mode 100644 index 0000000000..ea616d9bdf --- /dev/null +++ b/sdk/python/test_data/pipelines/if_elif_else_complex.py @@ -0,0 +1,97 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +from kfp import compiler +from kfp import dsl + + +@dsl.component +def int_0_to_9999() -> int: + import random + return random.randint(0, 9999) + + +@dsl.component +def is_even_or_odd(num: int) -> str: + return 'odd' if num % 2 else 'even' + + +@dsl.component +def print_and_return(text: str) -> str: + print(text) + return text + + +@dsl.component +def print_strings(strings: List[str]): + print(strings) + + +@dsl.component +def print_ints(ints: List[int]): + print(ints) + + +@dsl.pipeline +def lucky_number_pipeline(add_drumroll: bool = True, + repeat_if_lucky_number: bool = True, + trials: List[int] = [1, 2, 3]): + with dsl.ParallelFor(trials) as trial: + int_task = int_0_to_9999().set_caching_options(False) + with dsl.If(add_drumroll == True): + with dsl.If(trial == 3): + print_and_return(text='Adding drumroll on last trial!') + + with dsl.If(int_task.output < 5000): + + even_or_odd_task = is_even_or_odd(num=int_task.output) + + with dsl.If(even_or_odd_task.output == 'even'): + t1 = print_and_return(text='Got a low even number!') + with dsl.Else(): + t2 = print_and_return(text='Got a low odd number!') + + repeater_task = print_and_return( + text=dsl.OneOf(t1.output, t2.output)) + + with dsl.Elif(int_task.output > 5000): + + even_or_odd_task = is_even_or_odd(num=int_task.output) + + with dsl.If(even_or_odd_task.output == 'even'): + t3 = print_and_return(text='Got a high even number!') + with dsl.Else(): + t4 = print_and_return(text='Got a high odd number!') + + repeater_task = print_and_return( + text=dsl.OneOf(t3.output, t4.output)) + + with dsl.Else(): + print_and_return( + text='Announcing: Got the lucky number 5000! A one in 10,000 chance.' + ) + with dsl.If(repeat_if_lucky_number == True): + with dsl.ParallelFor([1, 2]) as _: + print_and_return( + text='Announcing again: Got the lucky number 5000! A one in 10,000 chance.' + ) + + print_ints(ints=dsl.Collected(int_task.output)) + + +if __name__ == '__main__': + compiler.Compiler().compile( + pipeline_func=lucky_number_pipeline, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/if_elif_else_complex.yaml b/sdk/python/test_data/pipelines/if_elif_else_complex.yaml new file mode 100644 index 0000000000..b1f5520ba1 --- /dev/null +++ b/sdk/python/test_data/pipelines/if_elif_else_complex.yaml @@ -0,0 +1,1118 @@ +# PIPELINE DEFINITION +# Name: lucky-number-pipeline +# Inputs: +# add_drumroll: bool [Default: True] +# repeat_if_lucky_number: bool [Default: True] +# trials: list [Default: [1.0, 2.0, 3.0]] +components: + comp-condition-11: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-5-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return-5 + tasks: + print-and-return-5: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-5 + inputs: + parameters: + text: + runtimeValue: + constant: Got a high even number! + taskInfo: + name: print-and-return-5 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--is-even-or-odd-2-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-5-Output: + parameterType: STRING + comp-condition-12: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-6-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return-6 + tasks: + print-and-return-6: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-6 + inputs: + parameters: + text: + runtimeValue: + constant: Got a high odd number! + taskInfo: + name: print-and-return-6 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--is-even-or-odd-2-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-6-Output: + parameterType: STRING + comp-condition-13: + dag: + tasks: + condition-14: + componentRef: + name: comp-condition-14 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--repeat_if_lucky_number: + componentInputParameter: pipelinechannel--repeat_if_lucky_number + taskInfo: + name: condition-14 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--repeat_if_lucky_number'] + == true + print-and-return-8: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-8 + inputs: + parameters: + text: + runtimeValue: + constant: 'Announcing: Got the lucky number 5000! A one in 10,000 + chance.' + taskInfo: + name: print-and-return-8 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--repeat_if_lucky_number: + parameterType: BOOLEAN + comp-condition-14: + dag: + tasks: + for-loop-16: + componentRef: + name: comp-for-loop-16 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--repeat_if_lucky_number: + componentInputParameter: pipelinechannel--repeat_if_lucky_number + parameterIterator: + itemInput: pipelinechannel--loop-item-param-15 + items: + raw: '[1, 2]' + taskInfo: + name: for-loop-16 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--repeat_if_lucky_number: + parameterType: BOOLEAN + comp-condition-2: + dag: + tasks: + condition-3: + componentRef: + name: comp-condition-3 + inputs: + parameters: + pipelinechannel--add_drumroll: + componentInputParameter: pipelinechannel--add_drumroll + pipelinechannel--trials-loop-item: + componentInputParameter: pipelinechannel--trials-loop-item + taskInfo: + name: condition-3 + triggerPolicy: + condition: int(inputs.parameter_values['pipelinechannel--trials-loop-item']) + == 3 + inputDefinitions: + parameters: + pipelinechannel--add_drumroll: + parameterType: BOOLEAN + pipelinechannel--trials-loop-item: + parameterType: NUMBER_INTEGER + comp-condition-3: + dag: + tasks: + print-and-return: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return + inputs: + parameters: + text: + runtimeValue: + constant: Adding drumroll on last trial! + taskInfo: + name: print-and-return + inputDefinitions: + parameters: + pipelinechannel--add_drumroll: + parameterType: BOOLEAN + pipelinechannel--trials-loop-item: + parameterType: NUMBER_INTEGER + comp-condition-6: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-2-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return-2 + tasks: + print-and-return-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-2 + inputs: + parameters: + text: + runtimeValue: + constant: Got a low even number! + taskInfo: + name: print-and-return-2 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--is-even-or-odd-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-2-Output: + parameterType: STRING + comp-condition-7: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-3-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return-3 + tasks: + print-and-return-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-3 + inputs: + parameters: + text: + runtimeValue: + constant: Got a low odd number! + taskInfo: + name: print-and-return-3 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--is-even-or-odd-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-3-Output: + parameterType: STRING + comp-condition-8: + dag: + tasks: + condition-branches-5: + componentRef: + name: comp-condition-branches-5 + dependentTasks: + - is-even-or-odd + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--is-even-or-odd-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: is-even-or-odd + taskInfo: + name: condition-branches-5 + is-even-or-odd: + cachingOptions: + enableCache: true + componentRef: + name: comp-is-even-or-odd + inputs: + parameters: + num: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + taskInfo: + name: is-even-or-odd + print-and-return-4: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-4 + dependentTasks: + - condition-branches-5 + inputs: + parameters: + text: + taskOutputParameter: + outputParameterKey: pipelinechannel--condition-branches-5-oneof-1 + producerTask: condition-branches-5 + taskInfo: + name: print-and-return-4 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + comp-condition-9: + dag: + tasks: + condition-branches-10: + componentRef: + name: comp-condition-branches-10 + dependentTasks: + - is-even-or-odd-2 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--is-even-or-odd-2-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: is-even-or-odd-2 + taskInfo: + name: condition-branches-10 + is-even-or-odd-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-is-even-or-odd-2 + inputs: + parameters: + num: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + taskInfo: + name: is-even-or-odd-2 + print-and-return-7: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-7 + dependentTasks: + - condition-branches-10 + inputs: + parameters: + text: + taskOutputParameter: + outputParameterKey: pipelinechannel--condition-branches-10-oneof-1 + producerTask: condition-branches-10 + taskInfo: + name: print-and-return-7 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + comp-condition-branches-10: + dag: + outputs: + parameters: + pipelinechannel--condition-branches-10-oneof-1: + valueFromOneof: + parameterSelectors: + - outputParameterKey: pipelinechannel--print-and-return-5-Output + producerSubtask: condition-11 + - outputParameterKey: pipelinechannel--print-and-return-6-Output + producerSubtask: condition-12 + tasks: + condition-11: + componentRef: + name: comp-condition-11 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--is-even-or-odd-2-Output: + componentInputParameter: pipelinechannel--is-even-or-odd-2-Output + taskInfo: + name: condition-11 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--is-even-or-odd-2-Output'] + == 'even' + condition-12: + componentRef: + name: comp-condition-12 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--is-even-or-odd-2-Output: + componentInputParameter: pipelinechannel--is-even-or-odd-2-Output + taskInfo: + name: condition-12 + triggerPolicy: + condition: '!(inputs.parameter_values[''pipelinechannel--is-even-or-odd-2-Output''] + == ''even'')' + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--is-even-or-odd-2-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--condition-branches-10-oneof-1: + parameterType: STRING + comp-condition-branches-4: + dag: + tasks: + condition-13: + componentRef: + name: comp-condition-13 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--repeat_if_lucky_number: + componentInputParameter: pipelinechannel--repeat_if_lucky_number + taskInfo: + name: condition-13 + triggerPolicy: + condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + < 5000) && !(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + > 5000)' + condition-8: + componentRef: + name: comp-condition-8 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + taskInfo: + name: condition-8 + triggerPolicy: + condition: int(inputs.parameter_values['pipelinechannel--int-0-to-9999-Output']) + < 5000 + condition-9: + componentRef: + name: comp-condition-9 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + taskInfo: + name: condition-9 + triggerPolicy: + condition: '!(int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + < 5000) && int(inputs.parameter_values[''pipelinechannel--int-0-to-9999-Output'']) + > 5000' + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--repeat_if_lucky_number: + parameterType: BOOLEAN + comp-condition-branches-5: + dag: + outputs: + parameters: + pipelinechannel--condition-branches-5-oneof-1: + valueFromOneof: + parameterSelectors: + - outputParameterKey: pipelinechannel--print-and-return-2-Output + producerSubtask: condition-6 + - outputParameterKey: pipelinechannel--print-and-return-3-Output + producerSubtask: condition-7 + tasks: + condition-6: + componentRef: + name: comp-condition-6 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--is-even-or-odd-Output: + componentInputParameter: pipelinechannel--is-even-or-odd-Output + taskInfo: + name: condition-6 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--is-even-or-odd-Output'] + == 'even' + condition-7: + componentRef: + name: comp-condition-7 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + componentInputParameter: pipelinechannel--int-0-to-9999-Output + pipelinechannel--is-even-or-odd-Output: + componentInputParameter: pipelinechannel--is-even-or-odd-Output + taskInfo: + name: condition-7 + triggerPolicy: + condition: '!(inputs.parameter_values[''pipelinechannel--is-even-or-odd-Output''] + == ''even'')' + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--is-even-or-odd-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--condition-branches-5-oneof-1: + parameterType: STRING + comp-for-loop-1: + dag: + outputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: int-0-to-9999 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + inputs: + parameters: + pipelinechannel--add_drumroll: + componentInputParameter: pipelinechannel--add_drumroll + pipelinechannel--trials-loop-item: + componentInputParameter: pipelinechannel--trials-loop-item + taskInfo: + name: condition-2 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--add_drumroll'] == + true + condition-branches-4: + componentRef: + name: comp-condition-branches-4 + dependentTasks: + - int-0-to-9999 + inputs: + parameters: + pipelinechannel--int-0-to-9999-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: int-0-to-9999 + pipelinechannel--repeat_if_lucky_number: + componentInputParameter: pipelinechannel--repeat_if_lucky_number + taskInfo: + name: condition-branches-4 + int-0-to-9999: + cachingOptions: {} + componentRef: + name: comp-int-0-to-9999 + taskInfo: + name: int-0-to-9999 + inputDefinitions: + parameters: + pipelinechannel--add_drumroll: + parameterType: BOOLEAN + pipelinechannel--repeat_if_lucky_number: + parameterType: BOOLEAN + pipelinechannel--trials: + parameterType: LIST + pipelinechannel--trials-loop-item: + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: LIST + comp-for-loop-16: + dag: + tasks: + print-and-return-9: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-9 + inputs: + parameters: + text: + runtimeValue: + constant: 'Announcing again: Got the lucky number 5000! A one in + 10,000 chance.' + taskInfo: + name: print-and-return-9 + inputDefinitions: + parameters: + pipelinechannel--int-0-to-9999-Output: + parameterType: NUMBER_INTEGER + pipelinechannel--loop-item-param-15: + parameterType: NUMBER_INTEGER + pipelinechannel--repeat_if_lucky_number: + parameterType: BOOLEAN + comp-int-0-to-9999: + executorLabel: exec-int-0-to-9999 + outputDefinitions: + parameters: + Output: + parameterType: NUMBER_INTEGER + comp-is-even-or-odd: + executorLabel: exec-is-even-or-odd + inputDefinitions: + parameters: + num: + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-is-even-or-odd-2: + executorLabel: exec-is-even-or-odd-2 + inputDefinitions: + parameters: + num: + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return: + executorLabel: exec-print-and-return + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-2: + executorLabel: exec-print-and-return-2 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-3: + executorLabel: exec-print-and-return-3 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-4: + executorLabel: exec-print-and-return-4 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-5: + executorLabel: exec-print-and-return-5 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-6: + executorLabel: exec-print-and-return-6 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-7: + executorLabel: exec-print-and-return-7 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-8: + executorLabel: exec-print-and-return-8 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-9: + executorLabel: exec-print-and-return-9 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-ints: + executorLabel: exec-print-ints + inputDefinitions: + parameters: + ints: + parameterType: LIST +deploymentSpec: + executors: + exec-int-0-to-9999: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - int_0_to_9999 + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef int_0_to_9999() -> int:\n import random\n return random.randint(0,\ + \ 9999)\n\n" + image: python:3.7 + exec-is-even-or-odd: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - is_even_or_odd + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef is_even_or_odd(num: int) -> str:\n return 'odd' if num % 2\ + \ else 'even'\n\n" + image: python:3.7 + exec-is-even-or-odd-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - is_even_or_odd + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef is_even_or_odd(num: int) -> str:\n return 'odd' if num % 2\ + \ else 'even'\n\n" + image: python:3.7 + exec-print-and-return: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-4: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-5: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-6: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-7: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-8: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-9: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-ints: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_ints + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_ints(ints: List[int]):\n print(ints)\n\n" + image: python:3.7 +pipelineInfo: + name: lucky-number-pipeline +root: + dag: + tasks: + for-loop-1: + componentRef: + name: comp-for-loop-1 + inputs: + parameters: + pipelinechannel--add_drumroll: + componentInputParameter: add_drumroll + pipelinechannel--repeat_if_lucky_number: + componentInputParameter: repeat_if_lucky_number + pipelinechannel--trials: + componentInputParameter: trials + parameterIterator: + itemInput: pipelinechannel--trials-loop-item + items: + inputParameter: pipelinechannel--trials + taskInfo: + name: for-loop-1 + print-ints: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-ints + dependentTasks: + - for-loop-1 + inputs: + parameters: + ints: + taskOutputParameter: + outputParameterKey: pipelinechannel--int-0-to-9999-Output + producerTask: for-loop-1 + taskInfo: + name: print-ints + inputDefinitions: + parameters: + add_drumroll: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + repeat_if_lucky_number: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + trials: + defaultValue: + - 1.0 + - 2.0 + - 3.0 + isOptional: true + parameterType: LIST +schemaVersion: 2.1.0 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/if_elif_else_with_oneof_parameters.py b/sdk/python/test_data/pipelines/if_elif_else_with_oneof_parameters.py new file mode 100644 index 0000000000..7e0dc1b57f --- /dev/null +++ b/sdk/python/test_data/pipelines/if_elif_else_with_oneof_parameters.py @@ -0,0 +1,65 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from kfp import compiler +from kfp import dsl + + +@dsl.component +def flip_three_sided_die() -> str: + import random + val = random.randint(0, 2) + + if val == 0: + return 'heads' + elif val == 1: + return 'tails' + else: + return 'draw' + + +@dsl.component +def print_and_return(text: str) -> str: + print(text) + return text + + +@dsl.component +def special_print_and_return(text: str, output_key: dsl.OutputPath(str)): + print('Got the special state:', text) + with open(output_key, 'w') as f: + f.write(text) + + +@dsl.pipeline +def roll_die_pipeline() -> str: + flip_coin_task = flip_three_sided_die() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = print_and_return(text='Got heads!') + with dsl.Elif(flip_coin_task.output == 'tails'): + t2 = print_and_return(text='Got tails!') + with dsl.Else(): + t3 = special_print_and_return(text='Draw!') + return dsl.OneOf(t1.output, t2.output, t3.outputs['output_key']) + + +@dsl.pipeline +def outer_pipeline() -> str: + flip_coin_task = roll_die_pipeline() + return print_and_return(text=flip_coin_task.output).output + + +if __name__ == '__main__': + compiler.Compiler().compile( + pipeline_func=outer_pipeline, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/if_elif_else_with_oneof_parameters.yaml b/sdk/python/test_data/pipelines/if_elif_else_with_oneof_parameters.yaml new file mode 100644 index 0000000000..0915994760 --- /dev/null +++ b/sdk/python/test_data/pipelines/if_elif_else_with_oneof_parameters.yaml @@ -0,0 +1,420 @@ +# PIPELINE DEFINITION +# Name: outer-pipeline +# Outputs: +# Output: str +components: + comp-condition-2: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return + tasks: + print-and-return: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return + inputs: + parameters: + text: + runtimeValue: + constant: Got heads! + taskInfo: + name: print-and-return + inputDefinitions: + parameters: + pipelinechannel--flip-three-sided-die-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-Output: + parameterType: STRING + comp-condition-3: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-2-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return-2 + tasks: + print-and-return-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-2 + inputs: + parameters: + text: + runtimeValue: + constant: Got tails! + taskInfo: + name: print-and-return-2 + inputDefinitions: + parameters: + pipelinechannel--flip-three-sided-die-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-2-Output: + parameterType: STRING + comp-condition-4: + dag: + outputs: + parameters: + pipelinechannel--special-print-and-return-output_key: + valueFromParameter: + outputParameterKey: output_key + producerSubtask: special-print-and-return + tasks: + special-print-and-return: + cachingOptions: + enableCache: true + componentRef: + name: comp-special-print-and-return + inputs: + parameters: + text: + runtimeValue: + constant: Draw! + taskInfo: + name: special-print-and-return + inputDefinitions: + parameters: + pipelinechannel--flip-three-sided-die-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--special-print-and-return-output_key: + parameterType: STRING + comp-condition-branches-1: + dag: + outputs: + parameters: + pipelinechannel--condition-branches-1-oneof-1: + valueFromOneof: + parameterSelectors: + - outputParameterKey: pipelinechannel--print-and-return-Output + producerSubtask: condition-2 + - outputParameterKey: pipelinechannel--print-and-return-2-Output + producerSubtask: condition-3 + - outputParameterKey: pipelinechannel--special-print-and-return-output_key + producerSubtask: condition-4 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + inputs: + parameters: + pipelinechannel--flip-three-sided-die-Output: + componentInputParameter: pipelinechannel--flip-three-sided-die-Output + taskInfo: + name: condition-2 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--flip-three-sided-die-Output'] + == 'heads' + condition-3: + componentRef: + name: comp-condition-3 + inputs: + parameters: + pipelinechannel--flip-three-sided-die-Output: + componentInputParameter: pipelinechannel--flip-three-sided-die-Output + taskInfo: + name: condition-3 + triggerPolicy: + condition: '!(inputs.parameter_values[''pipelinechannel--flip-three-sided-die-Output''] + == ''heads'') && inputs.parameter_values[''pipelinechannel--flip-three-sided-die-Output''] + == ''tails''' + condition-4: + componentRef: + name: comp-condition-4 + inputs: + parameters: + pipelinechannel--flip-three-sided-die-Output: + componentInputParameter: pipelinechannel--flip-three-sided-die-Output + taskInfo: + name: condition-4 + triggerPolicy: + condition: '!(inputs.parameter_values[''pipelinechannel--flip-three-sided-die-Output''] + == ''heads'') && !(inputs.parameter_values[''pipelinechannel--flip-three-sided-die-Output''] + == ''tails'')' + inputDefinitions: + parameters: + pipelinechannel--flip-three-sided-die-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--condition-branches-1-oneof-1: + parameterType: STRING + comp-flip-three-sided-die: + executorLabel: exec-flip-three-sided-die + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return: + executorLabel: exec-print-and-return + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-2: + executorLabel: exec-print-and-return-2 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-3: + executorLabel: exec-print-and-return-3 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-roll-die-pipeline: + dag: + outputs: + parameters: + Output: + valueFromParameter: + outputParameterKey: pipelinechannel--condition-branches-1-oneof-1 + producerSubtask: condition-branches-1 + tasks: + condition-branches-1: + componentRef: + name: comp-condition-branches-1 + dependentTasks: + - flip-three-sided-die + inputs: + parameters: + pipelinechannel--flip-three-sided-die-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: flip-three-sided-die + taskInfo: + name: condition-branches-1 + flip-three-sided-die: + cachingOptions: + enableCache: true + componentRef: + name: comp-flip-three-sided-die + taskInfo: + name: flip-three-sided-die + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-special-print-and-return: + executorLabel: exec-special-print-and-return + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + output_key: + parameterType: STRING +deploymentSpec: + executors: + exec-flip-three-sided-die: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - flip_three_sided_die + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef flip_three_sided_die() -> str:\n import random\n val =\ + \ random.randint(0, 2)\n\n if val == 0:\n return 'heads'\n \ + \ elif val == 1:\n return 'tails'\n else:\n return 'draw'\n\ + \n" + image: python:3.7 + exec-print-and-return: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-special-print-and-return: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - special_print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef special_print_and_return(text: str, output_key: dsl.OutputPath(str)):\n\ + \ print('Got the special state:', text)\n with open(output_key, 'w')\ + \ as f:\n f.write(text)\n\n" + image: python:3.7 +pipelineInfo: + name: outer-pipeline +root: + dag: + outputs: + parameters: + Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return + tasks: + print-and-return: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-3 + dependentTasks: + - roll-die-pipeline + inputs: + parameters: + text: + taskOutputParameter: + outputParameterKey: Output + producerTask: roll-die-pipeline + taskInfo: + name: print-and-return + roll-die-pipeline: + cachingOptions: + enableCache: true + componentRef: + name: comp-roll-die-pipeline + taskInfo: + name: roll-die-pipeline + outputDefinitions: + parameters: + Output: + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/if_else_with_oneof_artifacts.py b/sdk/python/test_data/pipelines/if_else_with_oneof_artifacts.py new file mode 100644 index 0000000000..4dc549c173 --- /dev/null +++ b/sdk/python/test_data/pipelines/if_else_with_oneof_artifacts.py @@ -0,0 +1,60 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Input +from kfp.dsl import Output + + +@dsl.component +def flip_coin() -> str: + import random + return 'heads' if random.randint(0, 1) == 0 else 'tails' + + +@dsl.component +def param_to_artifact(val: str, a: Output[Artifact]): + with open(a.path, 'w') as f: + f.write(val) + + +@dsl.component +def print_artifact(a: Input[Artifact]): + with open(a.path) as f: + print(f.read()) + + +@dsl.pipeline +def flip_coin_pipeline() -> Artifact: + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + t1 = param_to_artifact(val=flip_coin_task.output) + with dsl.Else(): + t2 = param_to_artifact(val=flip_coin_task.output) + oneof = dsl.OneOf(t1.outputs['a'], t2.outputs['a']) + print_artifact(a=oneof) + return oneof + + +@dsl.pipeline +def outer_pipeline(): + flip_coin_task = flip_coin_pipeline() + print_artifact(a=flip_coin_task.output) + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=outer_pipeline, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/if_else_with_oneof_artifacts.yaml b/sdk/python/test_data/pipelines/if_else_with_oneof_artifacts.yaml new file mode 100644 index 0000000000..89e2a659fa --- /dev/null +++ b/sdk/python/test_data/pipelines/if_else_with_oneof_artifacts.yaml @@ -0,0 +1,380 @@ +# PIPELINE DEFINITION +# Name: outer-pipeline +components: + comp-condition-2: + dag: + outputs: + artifacts: + pipelinechannel--param-to-artifact-a: + artifactSelectors: + - outputArtifactKey: a + producerSubtask: param-to-artifact + tasks: + param-to-artifact: + cachingOptions: + enableCache: true + componentRef: + name: comp-param-to-artifact + inputs: + parameters: + val: + componentInputParameter: pipelinechannel--flip-coin-Output + taskInfo: + name: param-to-artifact + inputDefinitions: + parameters: + pipelinechannel--flip-coin-Output: + parameterType: STRING + outputDefinitions: + artifacts: + pipelinechannel--param-to-artifact-a: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-condition-3: + dag: + outputs: + artifacts: + pipelinechannel--param-to-artifact-2-a: + artifactSelectors: + - outputArtifactKey: a + producerSubtask: param-to-artifact-2 + tasks: + param-to-artifact-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-param-to-artifact-2 + inputs: + parameters: + val: + componentInputParameter: pipelinechannel--flip-coin-Output + taskInfo: + name: param-to-artifact-2 + inputDefinitions: + parameters: + pipelinechannel--flip-coin-Output: + parameterType: STRING + outputDefinitions: + artifacts: + pipelinechannel--param-to-artifact-2-a: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-condition-branches-1: + dag: + outputs: + artifacts: + pipelinechannel--condition-branches-1-oneof-1: + artifactSelectors: + - outputArtifactKey: pipelinechannel--param-to-artifact-a + producerSubtask: condition-2 + - outputArtifactKey: pipelinechannel--param-to-artifact-2-a + producerSubtask: condition-3 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + inputs: + parameters: + pipelinechannel--flip-coin-Output: + componentInputParameter: pipelinechannel--flip-coin-Output + taskInfo: + name: condition-2 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--flip-coin-Output'] + == 'heads' + condition-3: + componentRef: + name: comp-condition-3 + inputs: + parameters: + pipelinechannel--flip-coin-Output: + componentInputParameter: pipelinechannel--flip-coin-Output + taskInfo: + name: condition-3 + triggerPolicy: + condition: '!(inputs.parameter_values[''pipelinechannel--flip-coin-Output''] + == ''heads'')' + inputDefinitions: + parameters: + pipelinechannel--flip-coin-Output: + parameterType: STRING + outputDefinitions: + artifacts: + pipelinechannel--condition-branches-1-oneof-1: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-flip-coin: + executorLabel: exec-flip-coin + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-flip-coin-pipeline: + dag: + outputs: + artifacts: + Output: + artifactSelectors: + - outputArtifactKey: pipelinechannel--condition-branches-1-oneof-1 + producerSubtask: condition-branches-1 + tasks: + condition-branches-1: + componentRef: + name: comp-condition-branches-1 + dependentTasks: + - flip-coin + inputs: + parameters: + pipelinechannel--flip-coin-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: flip-coin + taskInfo: + name: condition-branches-1 + flip-coin: + cachingOptions: + enableCache: true + componentRef: + name: comp-flip-coin + taskInfo: + name: flip-coin + print-artifact: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-artifact + dependentTasks: + - condition-branches-1 + inputs: + artifacts: + a: + taskOutputArtifact: + outputArtifactKey: pipelinechannel--condition-branches-1-oneof-1 + producerTask: condition-branches-1 + taskInfo: + name: print-artifact + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-param-to-artifact: + executorLabel: exec-param-to-artifact + inputDefinitions: + parameters: + val: + parameterType: STRING + outputDefinitions: + artifacts: + a: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-param-to-artifact-2: + executorLabel: exec-param-to-artifact-2 + inputDefinitions: + parameters: + val: + parameterType: STRING + outputDefinitions: + artifacts: + a: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-print-artifact: + executorLabel: exec-print-artifact + inputDefinitions: + artifacts: + a: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-print-artifact-2: + executorLabel: exec-print-artifact-2 + inputDefinitions: + artifacts: + a: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-flip-coin: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - flip_coin + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef flip_coin() -> str:\n import random\n return 'heads' if\ + \ random.randint(0, 1) == 0 else 'tails'\n\n" + image: python:3.7 + exec-param-to-artifact: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - param_to_artifact + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef param_to_artifact(val: str, a: Output[Artifact]):\n with open(a.path,\ + \ 'w') as f:\n f.write(val)\n\n" + image: python:3.7 + exec-param-to-artifact-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - param_to_artifact + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef param_to_artifact(val: str, a: Output[Artifact]):\n with open(a.path,\ + \ 'w') as f:\n f.write(val)\n\n" + image: python:3.7 + exec-print-artifact: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_artifact + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_artifact(a: Input[Artifact]):\n with open(a.path) as\ + \ f:\n print(f.read())\n\n" + image: python:3.7 + exec-print-artifact-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_artifact + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_artifact(a: Input[Artifact]):\n with open(a.path) as\ + \ f:\n print(f.read())\n\n" + image: python:3.7 +pipelineInfo: + name: outer-pipeline +root: + dag: + tasks: + flip-coin-pipeline: + cachingOptions: + enableCache: true + componentRef: + name: comp-flip-coin-pipeline + taskInfo: + name: flip-coin-pipeline + print-artifact: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-artifact-2 + dependentTasks: + - flip-coin-pipeline + inputs: + artifacts: + a: + taskOutputArtifact: + outputArtifactKey: Output + producerTask: flip-coin-pipeline + taskInfo: + name: print-artifact +schemaVersion: 2.1.0 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/if_else_with_oneof_parameters.py b/sdk/python/test_data/pipelines/if_else_with_oneof_parameters.py new file mode 100644 index 0000000000..05f7f93403 --- /dev/null +++ b/sdk/python/test_data/pipelines/if_else_with_oneof_parameters.py @@ -0,0 +1,45 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from kfp import dsl + + +@dsl.component +def flip_coin() -> str: + import random + return 'heads' if random.randint(0, 1) == 0 else 'tails' + + +@dsl.component +def print_and_return(text: str) -> str: + print(text) + return text + + +@dsl.pipeline +def flip_coin_pipeline() -> str: + flip_coin_task = flip_coin() + with dsl.If(flip_coin_task.output == 'heads'): + print_task_1 = print_and_return(text='Got heads!') + with dsl.Else(): + print_task_2 = print_and_return(text='Got tails!') + x = dsl.OneOf(print_task_1.output, print_task_2.output) + print_and_return(text=x) + return x + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=flip_coin_pipeline, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/if_else_with_oneof_parameters.yaml b/sdk/python/test_data/pipelines/if_else_with_oneof_parameters.yaml new file mode 100644 index 0000000000..873288dd7e --- /dev/null +++ b/sdk/python/test_data/pipelines/if_else_with_oneof_parameters.yaml @@ -0,0 +1,313 @@ +# PIPELINE DEFINITION +# Name: flip-coin-pipeline +# Outputs: +# Output: str +components: + comp-condition-2: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return + tasks: + print-and-return: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return + inputs: + parameters: + text: + runtimeValue: + constant: Got heads! + taskInfo: + name: print-and-return + inputDefinitions: + parameters: + pipelinechannel--flip-coin-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-Output: + parameterType: STRING + comp-condition-3: + dag: + outputs: + parameters: + pipelinechannel--print-and-return-2-Output: + valueFromParameter: + outputParameterKey: Output + producerSubtask: print-and-return-2 + tasks: + print-and-return-2: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-2 + inputs: + parameters: + text: + runtimeValue: + constant: Got tails! + taskInfo: + name: print-and-return-2 + inputDefinitions: + parameters: + pipelinechannel--flip-coin-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--print-and-return-2-Output: + parameterType: STRING + comp-condition-branches-1: + dag: + outputs: + parameters: + pipelinechannel--condition-branches-1-oneof-1: + valueFromOneof: + parameterSelectors: + - outputParameterKey: pipelinechannel--print-and-return-Output + producerSubtask: condition-2 + - outputParameterKey: pipelinechannel--print-and-return-2-Output + producerSubtask: condition-3 + tasks: + condition-2: + componentRef: + name: comp-condition-2 + inputs: + parameters: + pipelinechannel--flip-coin-Output: + componentInputParameter: pipelinechannel--flip-coin-Output + taskInfo: + name: condition-2 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--flip-coin-Output'] + == 'heads' + condition-3: + componentRef: + name: comp-condition-3 + inputs: + parameters: + pipelinechannel--flip-coin-Output: + componentInputParameter: pipelinechannel--flip-coin-Output + taskInfo: + name: condition-3 + triggerPolicy: + condition: '!(inputs.parameter_values[''pipelinechannel--flip-coin-Output''] + == ''heads'')' + inputDefinitions: + parameters: + pipelinechannel--flip-coin-Output: + parameterType: STRING + outputDefinitions: + parameters: + pipelinechannel--condition-branches-1-oneof-1: + parameterType: STRING + comp-flip-coin: + executorLabel: exec-flip-coin + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return: + executorLabel: exec-print-and-return + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-2: + executorLabel: exec-print-and-return-2 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING + comp-print-and-return-3: + executorLabel: exec-print-and-return-3 + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + parameters: + Output: + parameterType: STRING +deploymentSpec: + executors: + exec-flip-coin: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - flip_coin + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef flip_coin() -> str:\n import random\n return 'heads' if\ + \ random.randint(0, 1) == 0 else 'tails'\n\n" + image: python:3.7 + exec-print-and-return: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 + exec-print-and-return-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_and_return + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_and_return(text: str) -> str:\n print(text)\n return\ + \ text\n\n" + image: python:3.7 +pipelineInfo: + name: flip-coin-pipeline +root: + dag: + outputs: + parameters: + Output: + valueFromParameter: + outputParameterKey: pipelinechannel--condition-branches-1-oneof-1 + producerSubtask: condition-branches-1 + tasks: + condition-branches-1: + componentRef: + name: comp-condition-branches-1 + dependentTasks: + - flip-coin + inputs: + parameters: + pipelinechannel--flip-coin-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: flip-coin + taskInfo: + name: condition-branches-1 + flip-coin: + cachingOptions: + enableCache: true + componentRef: + name: comp-flip-coin + taskInfo: + name: flip-coin + print-and-return-3: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-and-return-3 + dependentTasks: + - condition-branches-1 + inputs: + parameters: + text: + taskOutputParameter: + outputParameterKey: pipelinechannel--condition-branches-1-oneof-1 + producerTask: condition-branches-1 + taskInfo: + name: print-and-return-3 + outputDefinitions: + parameters: + Output: + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.py b/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.py index fc0866d480..e0dc235e37 100644 --- a/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.py +++ b/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.py @@ -120,7 +120,7 @@ def train( model.metadata['accuracy'] = 0.9 -@dsl.pipeline(pipeline_root='dummy_root', name='my-test-pipeline-beta') +@dsl.pipeline(name='my-test-pipeline-beta') def pipeline(message: str, input_dict: Dict[str, int] = {'A': 1, 'B': 2}): preprocess_task = preprocess( diff --git a/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.yaml b/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.yaml index 8d262365f4..8b28ec4cd0 100644 --- a/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.yaml +++ b/sdk/python/test_data/pipelines/lightweight_python_functions_pipeline.yaml @@ -64,7 +64,6 @@ components: artifactType: schemaTitle: system.Model schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-preprocess: @@ -79,15 +78,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -131,15 +132,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -239,4 +242,4 @@ root: message: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.py b/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.py index 9681c48868..42fb9c87eb 100644 --- a/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.py +++ b/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.py @@ -67,7 +67,7 @@ def output_named_tuple( return output(scalar, metrics, model) -@dsl.pipeline(pipeline_root='dummy_root', name='functions-with-outputs') +@dsl.pipeline(name='functions-with-outputs') def pipeline(first_message: str, second_message: str, first_number: int, second_number: int): concat_op = concat_message(first=first_message, second=second_message) diff --git a/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.yaml b/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.yaml index f1146dff2b..ac8c2d12bb 100644 --- a/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.yaml +++ b/sdk/python/test_data/pipelines/lightweight_python_functions_with_outputs.yaml @@ -67,7 +67,6 @@ components: parameters: scalar: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-add-numbers: @@ -82,15 +81,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -109,15 +110,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -136,15 +139,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -163,15 +168,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -274,4 +281,4 @@ root: schemaTitle: system.Metrics schemaVersion: 0.0.1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_complex.yaml b/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_complex.yaml index ede6a1e7dd..e85d521966 100644 --- a/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_complex.yaml +++ b/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_complex.yaml @@ -285,15 +285,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -315,15 +317,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -345,15 +349,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -375,15 +381,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -403,15 +411,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -484,4 +494,4 @@ root: schemaTitle: system.Dataset schemaVersion: 0.0.1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.14 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_simple.yaml b/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_simple.yaml index 3526dcfd73..5e38c9cb30 100644 --- a/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_simple.yaml +++ b/sdk/python/test_data/pipelines/parallelfor_fan_in/artifacts_simple.yaml @@ -90,15 +90,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -136,15 +138,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -209,4 +213,4 @@ root: schemaVersion: 0.0.1 isArtifactList: true schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.14 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/parallelfor_fan_in/conditional_producer_and_consumers.yaml b/sdk/python/test_data/pipelines/parallelfor_fan_in/conditional_producer_and_consumers.yaml index 7a3cbbf6de..6cf59971e9 100644 --- a/sdk/python/test_data/pipelines/parallelfor_fan_in/conditional_producer_and_consumers.yaml +++ b/sdk/python/test_data/pipelines/parallelfor_fan_in/conditional_producer_and_consumers.yaml @@ -44,7 +44,7 @@ components: outputDefinitions: parameters: pipelinechannel--double-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-condition-4: dag: outputs: @@ -74,7 +74,7 @@ components: outputDefinitions: parameters: pipelinechannel--add-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-double: executorLabel: exec-double inputDefinitions: @@ -117,7 +117,7 @@ components: outputDefinitions: parameters: pipelinechannel--double-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST deploymentSpec: executors: exec-add: @@ -132,15 +132,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -158,15 +160,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -225,4 +229,4 @@ root: Output: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.14 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/parallelfor_fan_in/nested_with_parameters.yaml b/sdk/python/test_data/pipelines/parallelfor_fan_in/nested_with_parameters.yaml index a318183a26..ffefb1fac6 100644 --- a/sdk/python/test_data/pipelines/parallelfor_fan_in/nested_with_parameters.yaml +++ b/sdk/python/test_data/pipelines/parallelfor_fan_in/nested_with_parameters.yaml @@ -74,7 +74,7 @@ components: outputDefinitions: parameters: pipelinechannel--add-two-nums-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-for-loop-4: dag: outputs: @@ -135,7 +135,7 @@ components: outputDefinitions: parameters: pipelinechannel--add-two-nums-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST deploymentSpec: executors: exec-add: @@ -150,20 +150,22 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef add(nums: List[int]) -> int:\n import itertools\n return\ - \ sum(itertools.chain(*nums))\n\n" + \ *\n\ndef add(nums: List[List[int]]) -> int:\n import itertools\n \ + \ return sum(itertools.chain(*nums))\n\n" image: python:3.7 exec-add-two-nums: container: @@ -177,15 +179,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -203,15 +207,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -229,15 +235,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -283,4 +291,4 @@ root: Output: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.14 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_complex.yaml b/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_complex.yaml index c056ae02ba..9e2c0288af 100644 --- a/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_complex.yaml +++ b/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_complex.yaml @@ -90,9 +90,9 @@ components: outputDefinitions: parameters: pipelinechannel--double-2-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST pipelinechannel--double-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-for-loop-4: dag: outputs: @@ -120,7 +120,7 @@ components: outputDefinitions: parameters: pipelinechannel--double-2-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-for-loop-6: dag: outputs: @@ -167,9 +167,9 @@ components: outputDefinitions: parameters: pipelinechannel--nested-add-2-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST pipelinechannel--simple-add-2-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-nested-add: executorLabel: exec-nested-add inputDefinitions: @@ -224,15 +224,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -251,15 +253,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -277,15 +281,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -303,15 +309,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -330,15 +338,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -357,15 +367,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -383,15 +395,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -477,4 +491,4 @@ root: Output: parameterType: NUMBER_INTEGER schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.14 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_simple.yaml b/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_simple.yaml index f6c8694c0b..c315ffee16 100644 --- a/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_simple.yaml +++ b/sdk/python/test_data/pipelines/parallelfor_fan_in/parameters_simple.yaml @@ -60,7 +60,7 @@ components: outputDefinitions: parameters: pipelinechannel--double-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST deploymentSpec: executors: exec-add: @@ -75,15 +75,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -111,15 +113,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -180,4 +184,4 @@ root: Output: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.14 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/parallelfor_fan_in/pipeline_producer_consumer.yaml b/sdk/python/test_data/pipelines/parallelfor_fan_in/pipeline_producer_consumer.yaml index 1afb9904a0..2c6e4e4612 100644 --- a/sdk/python/test_data/pipelines/parallelfor_fan_in/pipeline_producer_consumer.yaml +++ b/sdk/python/test_data/pipelines/parallelfor_fan_in/pipeline_producer_consumer.yaml @@ -129,7 +129,7 @@ components: outputDefinitions: parameters: pipelinechannel--double-pipeline-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-for-loop-2-2: dag: outputs: @@ -157,7 +157,7 @@ components: outputDefinitions: parameters: pipelinechannel--echo-and-return-Output: - parameterType: STRING + parameterType: LIST comp-for-loop-4: dag: outputs: @@ -185,7 +185,7 @@ components: outputDefinitions: parameters: pipelinechannel--double-pipeline-Output: - parameterType: NUMBER_INTEGER + parameterType: LIST comp-join-and-print: executorLabel: exec-join-and-print inputDefinitions: @@ -206,15 +206,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -233,15 +235,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -259,15 +263,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -286,15 +292,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -356,4 +364,4 @@ root: Output: parameterType: NUMBER_INTEGER schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.14 +sdkVersion: kfp-2.3.0 diff --git a/sdk/python/test_data/pipelines/pipeline_as_exit_task.yaml b/sdk/python/test_data/pipelines/pipeline_as_exit_task.yaml index fdf3197f80..0280af0066 100644 --- a/sdk/python/test_data/pipelines/pipeline_as_exit_task.yaml +++ b/sdk/python/test_data/pipelines/pipeline_as_exit_task.yaml @@ -129,15 +129,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -156,15 +158,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -183,15 +187,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -210,15 +216,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -262,4 +270,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.17 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_in_pipeline.yaml b/sdk/python/test_data/pipelines/pipeline_in_pipeline.yaml index d6f24930c1..4c587ca06f 100644 --- a/sdk/python/test_data/pipelines/pipeline_in_pipeline.yaml +++ b/sdk/python/test_data/pipelines/pipeline_in_pipeline.yaml @@ -74,15 +74,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -101,15 +103,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -152,4 +156,4 @@ root: taskInfo: name: print-op1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_in_pipeline_complex.yaml b/sdk/python/test_data/pipelines/pipeline_in_pipeline_complex.yaml index 248066037f..4f40dc08a5 100644 --- a/sdk/python/test_data/pipelines/pipeline_in_pipeline_complex.yaml +++ b/sdk/python/test_data/pipelines/pipeline_in_pipeline_complex.yaml @@ -161,15 +161,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -188,15 +190,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -241,4 +245,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_in_pipeline_loaded_from_yaml.yaml b/sdk/python/test_data/pipelines/pipeline_in_pipeline_loaded_from_yaml.yaml index 9758383cad..9d84ea8c08 100644 --- a/sdk/python/test_data/pipelines/pipeline_in_pipeline_loaded_from_yaml.yaml +++ b/sdk/python/test_data/pipelines/pipeline_in_pipeline_loaded_from_yaml.yaml @@ -152,15 +152,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -179,15 +181,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.14'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -206,15 +210,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -264,4 +270,4 @@ root: taskInfo: name: print-op1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_after.py b/sdk/python/test_data/pipelines/pipeline_with_after.py index a1093a4449..7ffd8015e3 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_after.py +++ b/sdk/python/test_data/pipelines/pipeline_with_after.py @@ -33,7 +33,7 @@ """) -@dsl.pipeline(name='pipeline-with-after', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-after') def my_pipeline(): task1 = component_op(text='1st task') task2 = component_op(text='2nd task').after(task1) diff --git a/sdk/python/test_data/pipelines/pipeline_with_after.yaml b/sdk/python/test_data/pipelines/pipeline_with_after.yaml index 84e4b213a3..8ddd829b36 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_after.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_after.yaml @@ -19,7 +19,6 @@ components: parameters: text: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-print-text: @@ -105,4 +104,4 @@ root: taskInfo: name: print-text-3 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.py b/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.py index 13bf9170b2..04b0723695 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.py +++ b/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.py @@ -23,9 +23,7 @@ str(test_data_dir / 'concat_placeholder_component.yaml')) -@dsl.pipeline( - name='one-step-pipeline-with-concat-placeholder', - pipeline_root='dummy_root') +@dsl.pipeline(name='one-step-pipeline-with-concat-placeholder') def my_pipeline(): component = component_op(input_prefix='some prefix:') diff --git a/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.yaml b/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.yaml index 124248a170..27fbeaf92c 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_concat_placeholder.yaml @@ -7,7 +7,6 @@ components: parameters: input_prefix: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-component-with-concat-placeholder: @@ -34,4 +33,4 @@ root: taskInfo: name: component-with-concat-placeholder schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_condition.py b/sdk/python/test_data/pipelines/pipeline_with_condition.py index a2c55c023f..da292a8261 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_condition.py +++ b/sdk/python/test_data/pipelines/pipeline_with_condition.py @@ -31,7 +31,7 @@ def print_op(msg: str): print(msg) -@dsl.pipeline(name='single-condition-pipeline', pipeline_root='dummy_root') +@dsl.pipeline(name='single-condition-pipeline') def my_pipeline(text: str = 'condition test'): flip1 = flip_coin_op().set_caching_options(False) print_op(msg=flip1.output) diff --git a/sdk/python/test_data/pipelines/pipeline_with_condition.yaml b/sdk/python/test_data/pipelines/pipeline_with_condition.yaml index 14c0fef350..10bc356118 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_condition.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_condition.yaml @@ -74,7 +74,6 @@ components: parameters: msg: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-flip-coin-op: @@ -89,15 +88,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -117,15 +118,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -145,15 +148,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -172,15 +177,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -199,15 +206,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -265,4 +274,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.py b/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.py index 099bd2d8dc..c988833f24 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.py +++ b/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.py @@ -27,7 +27,7 @@ def make_name(name: str) -> str: return name -@dsl.pipeline(name='pipeline-with-importer', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-importer') def my_pipeline(name: str = 'default-name', int_input: int = 1, pipeline_input_artifact_uri: str = DEFAULT_ARTIFACT_URI, diff --git a/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.yaml b/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.yaml index 1af374bad0..ecc8bdf199 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_dynamic_importer_metadata.yaml @@ -50,7 +50,6 @@ components: parameters: Output: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-importer: @@ -95,15 +94,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -182,4 +183,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_env.py b/sdk/python/test_data/pipelines/pipeline_with_env.py index 98ebd6d0cb..3b2ddf5de6 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_env.py +++ b/sdk/python/test_data/pipelines/pipeline_with_env.py @@ -44,7 +44,7 @@ def print_env_op(): """) -@dsl.pipeline(name='pipeline-with-env', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-env') def my_pipeline(): print_env_op().set_env_variable(name='ENV1', value='val1') print_env_2_op().set_env_variable( diff --git a/sdk/python/test_data/pipelines/pipeline_with_env.yaml b/sdk/python/test_data/pipelines/pipeline_with_env.yaml index 9e35173671..25d2f7cea8 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_env.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_env.yaml @@ -5,7 +5,6 @@ components: executorLabel: exec-print-env comp-print-env-op: executorLabel: exec-print-env-op -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-print-env: @@ -42,15 +41,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -80,4 +81,4 @@ root: taskInfo: name: print-env-op schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_exit_handler.yaml b/sdk/python/test_data/pipelines/pipeline_with_exit_handler.yaml index e3517d29d3..fdb340b1e7 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_exit_handler.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_exit_handler.yaml @@ -65,15 +65,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -92,15 +94,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -119,15 +123,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -171,4 +177,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.py b/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.py index 2deb7f0620..91ce665974 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.py +++ b/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.py @@ -25,7 +25,9 @@ def create_temporary_google_artifact_package( import os import textwrap - class VertexModel: + from kfp import dsl + + class VertexModel(dsl.Artifact): schema_title = 'google.VertexModel' schema_version = '0.0.0' @@ -38,7 +40,7 @@ def __init__(self, name: str, uri: str, metadata: dict) -> None: def path(self) -> str: return self.uri.replace('gs://', '/') - class VertexDataset: + class VertexDataset(dsl.Artifact): schema_title = 'google.VertexDataset' schema_version = '0.0.0' @@ -51,7 +53,7 @@ def __init__(self, name: str, uri: str, metadata: dict) -> None: def path(self) -> str: return self.uri.replace('gs://', '/') - class_source = textwrap.dedent( + class_source = 'from kfp import dsl' + '\n\n' + textwrap.dedent( inspect.getsource(VertexModel)) + '\n\n' + textwrap.dedent( inspect.getsource(VertexDataset)) diff --git a/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.yaml b/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.yaml index dfeab06f1f..63acac5901 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_google_artifact_type.yaml @@ -56,15 +56,18 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'aiplatform'\ - \ 'kfp==2.0.0-beta.16' 'kfp==2.0.0-beta.16' && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' &&\ + \ python3 -m pip install --quiet --no-warn-script-location 'aiplatform'\ + \ && \"$0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -89,15 +92,18 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'aiplatform'\ - \ 'kfp==2.0.0-beta.16' && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' &&\ + \ python3 -m pip install --quiet --no-warn-script-location 'aiplatform'\ + \ && \"$0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -150,4 +156,4 @@ root: taskInfo: name: model-producer schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.py b/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.py index eb221c6aff..4cc355c544 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.py +++ b/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.py @@ -23,8 +23,7 @@ str(test_data_dir / 'if_placeholder_component.yaml')) -@dsl.pipeline( - name='one-step-pipeline-with-if-placeholder', pipeline_root='dummy_root') +@dsl.pipeline(name='one-step-pipeline-with-if-placeholder') def my_pipeline(input0: str, input1: str, input2: str): # supply only optional_input_1 but not optional_input_2 component = component_op(required_input=input0, optional_input_1=input1) diff --git a/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.yaml b/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.yaml index 9a1285027b..ccf75514de 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_if_placeholder.yaml @@ -17,7 +17,6 @@ components: parameterType: STRING required_input: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-component-with-optional-inputs: @@ -56,4 +55,4 @@ root: input2: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_importer.py b/sdk/python/test_data/pipelines/pipeline_with_importer.py index 39bc647cd6..4ded772c76 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_importer.py +++ b/sdk/python/test_data/pipelines/pipeline_with_importer.py @@ -49,7 +49,7 @@ def pass_through_op(value: str) -> str: return value -@dsl.pipeline(name='pipeline-with-importer', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-importer') def my_pipeline(dataset2: str = 'gs://ml-pipeline-playground/shakespeare2.txt'): importer1 = importer( diff --git a/sdk/python/test_data/pipelines/pipeline_with_importer.yaml b/sdk/python/test_data/pipelines/pipeline_with_importer.yaml index dc34793aa9..ceb685de40 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_importer.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_importer.yaml @@ -96,7 +96,6 @@ components: parameters: scalar: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-importer: @@ -128,15 +127,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -160,15 +161,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -236,4 +239,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.py b/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.py index 98431d362c..3cbe9a488a 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.py +++ b/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.py @@ -38,8 +38,7 @@ class VertexDataset(dsl.Artifact): """) -@dsl.pipeline( - name='pipeline-with-importer-and-gcpc-type', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-importer-and-gcpc-type') def my_pipeline(): importer1 = importer( diff --git a/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.yaml b/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.yaml index aa47e4eef3..00dce2d86f 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_importer_and_gcpc_types.yaml @@ -21,7 +21,6 @@ components: artifactType: schemaTitle: google.VertexDataset schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-consumer-op: @@ -73,4 +72,4 @@ root: taskInfo: name: importer schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_loops.yaml b/sdk/python/test_data/pipelines/pipeline_with_loops.yaml index e4ea602990..61f902553a 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_loops.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_loops.yaml @@ -171,15 +171,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -198,15 +200,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -224,15 +228,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -250,15 +256,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -276,15 +284,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -302,15 +312,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -328,15 +340,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -354,15 +368,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -424,4 +440,4 @@ root: loop_parameter: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_loops_and_conditions.yaml b/sdk/python/test_data/pipelines/pipeline_with_loops_and_conditions.yaml index 31dc82fac3..ae099dcd54 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_loops_and_conditions.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_loops_and_conditions.yaml @@ -602,15 +602,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -631,15 +633,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -660,15 +664,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -688,15 +694,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -714,15 +722,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -741,15 +751,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -768,15 +780,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -795,15 +809,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -822,15 +838,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -849,15 +867,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -876,15 +896,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -903,15 +925,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -930,15 +954,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -1022,4 +1048,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_metadata_fields.yaml b/sdk/python/test_data/pipelines/pipeline_with_metadata_fields.yaml index 203ee18371..8b2b1a8f87 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_metadata_fields.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_metadata_fields.yaml @@ -60,15 +60,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -95,15 +97,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -172,4 +176,4 @@ root: schemaVersion: 0.0.1 description: The final concatenated dataset. schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.py b/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.py index 887f97c003..7b9992d36e 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.py +++ b/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.py @@ -28,7 +28,7 @@ def output_metrics(metrics: Output[Metrics]): metrics.log_metric('accuracy', result) -@dsl.pipeline(name='pipeline-with-metrics-outputs', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-metrics-outputs') def my_pipeline(): output_metrics() diff --git a/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.yaml b/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.yaml index db913faaf2..665041c65c 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_metrics_outputs.yaml @@ -46,7 +46,6 @@ components: artifactType: schemaTitle: system.Metrics schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-output-metrics: @@ -61,15 +60,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -90,15 +91,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -149,4 +152,4 @@ root: schemaTitle: system.Metrics schemaVersion: 0.0.1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_multiple_exit_handlers.yaml b/sdk/python/test_data/pipelines/pipeline_with_multiple_exit_handlers.yaml index 0006354da2..2207f0f022 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_multiple_exit_handlers.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_multiple_exit_handlers.yaml @@ -125,15 +125,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -152,15 +154,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -179,15 +183,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -206,15 +212,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -233,15 +241,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -260,15 +270,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -287,15 +299,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -389,4 +403,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_nested_conditions.yaml b/sdk/python/test_data/pipelines/pipeline_with_nested_conditions.yaml index ca51913681..817c7c76c2 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_nested_conditions.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_nested_conditions.yaml @@ -147,15 +147,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -175,15 +177,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -203,15 +207,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -231,15 +237,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -259,15 +267,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -286,15 +296,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -313,15 +325,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -340,15 +354,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -426,4 +442,4 @@ root: taskInfo: name: print-op-2 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.py b/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.py index 732aad76b0..89e22ed117 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.py +++ b/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.py @@ -68,7 +68,6 @@ def random_num_op(low, high): @dsl.pipeline( name='conditional-execution-pipeline', display_name='Conditional execution pipeline.', - pipeline_root='dummy_root', description='Shows how to use dsl.Condition().') def my_pipeline(): flip = flip_coin_op() diff --git a/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.yaml b/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.yaml index 153ccdb04d..d473abcd75 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_nested_conditions_yaml.yaml @@ -240,7 +240,6 @@ components: parameters: msg: parameterType: STRING -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-flip-coin: @@ -348,4 +347,4 @@ root: taskInfo: name: flip-coin schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_nested_loops.yaml b/sdk/python/test_data/pipelines/pipeline_with_nested_loops.yaml index 4bfb2efc39..279c732088 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_nested_loops.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_nested_loops.yaml @@ -1,7 +1,7 @@ # PIPELINE DEFINITION # Name: pipeline-with-nested-loops # Inputs: -# loop_parameter: list [Default: [{'p_b': 'hello', 'p_a': [{'q_a': '1'}, {'q_a': '2'}]}, {'p_b': 'halo', 'p_a': [{'q_a': '11'}, {'q_a': '22'}]}]] +# loop_parameter: list [Default: [{'p_a': [{'q_a': '1'}, {'q_a': '2'}], 'p_b': 'hello'}, {'p_a': [{'q_a': '11'}, {'q_a': '22'}], 'p_b': 'halo'}]] components: comp-for-loop-1: dag: @@ -145,15 +145,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -172,15 +174,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -199,15 +203,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -256,4 +262,4 @@ root: isOptional: true parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_ontology.py b/sdk/python/test_data/pipelines/pipeline_with_ontology.py index e85c3335cf..63dc89f214 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_ontology.py +++ b/sdk/python/test_data/pipelines/pipeline_with_ontology.py @@ -29,7 +29,6 @@ @dsl.pipeline( name='two-step-pipeline-with-ontology', - pipeline_root='dummy_root', description='A linear two-step pipeline with artifact ontology types.') def my_pipeline(input_location: str = 'gs://test-bucket/pipeline_root', optimizer: str = 'sgd', diff --git a/sdk/python/test_data/pipelines/pipeline_with_ontology.yaml b/sdk/python/test_data/pipelines/pipeline_with_ontology.yaml index 14760c3ee4..c738ba3515 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_ontology.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_ontology.yaml @@ -37,7 +37,6 @@ components: artifactType: schemaTitle: system.Model schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-ingestion: @@ -112,4 +111,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_outputs.yaml b/sdk/python/test_data/pipelines/pipeline_with_outputs.yaml index 99baf1b5f7..1815071bb5 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_outputs.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_outputs.yaml @@ -104,15 +104,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -131,15 +133,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -203,4 +207,4 @@ root: schemaTitle: system.Artifact schemaVersion: 0.0.1 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml index bd931cabb9..eaac51c057 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_parallelfor_parallelism.yaml @@ -179,15 +179,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -205,15 +207,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -231,15 +235,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -257,15 +263,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -283,15 +291,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -309,15 +319,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -357,4 +369,4 @@ root: loop_parameter: parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_params_containing_format.yaml b/sdk/python/test_data/pipelines/pipeline_with_params_containing_format.yaml index dfafe54771..60b1e7536a 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_params_containing_format.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_params_containing_format.yaml @@ -74,15 +74,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -101,15 +103,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -128,15 +132,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -201,4 +207,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_placeholders.yaml b/sdk/python/test_data/pipelines/pipeline_with_placeholders.yaml index ba86ea06ba..21ef1aafec 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_placeholders.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_placeholders.yaml @@ -55,15 +55,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -81,15 +83,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -107,15 +111,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -133,15 +139,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -159,15 +167,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -254,4 +264,4 @@ root: taskInfo: name: print-op-5 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_resource_spec.py b/sdk/python/test_data/pipelines/pipeline_with_resource_spec.py index c4a42119c6..29e1cc72ec 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_resource_spec.py +++ b/sdk/python/test_data/pipelines/pipeline_with_resource_spec.py @@ -29,7 +29,6 @@ @dsl.pipeline( name='two-step-pipeline-with-resource-spec', - pipeline_root='dummy_root', description='A linear two-step pipeline with resource specification.') def my_pipeline(input_location: str = 'gs://test-bucket/pipeline_root', optimizer: str = 'sgd', diff --git a/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml b/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml index 6448199c4f..e045b3c36b 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_resource_spec.yaml @@ -37,7 +37,6 @@ components: artifactType: schemaTitle: system.Model schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-ingestion: @@ -120,4 +119,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_retry.yaml b/sdk/python/test_data/pipelines/pipeline_with_retry.yaml index 3f6a05fc11..8e81c93350 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_retry.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_retry.yaml @@ -30,15 +30,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -78,4 +80,4 @@ root: isOptional: true parameterType: NUMBER_DOUBLE schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_reused_component.py b/sdk/python/test_data/pipelines/pipeline_with_reused_component.py index b824bff2c6..d52985bbe7 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_reused_component.py +++ b/sdk/python/test_data/pipelines/pipeline_with_reused_component.py @@ -23,7 +23,7 @@ str(test_data_dir / 'add_component.yaml')) -@dsl.pipeline(name='add-pipeline', pipeline_root='dummy_root') +@dsl.pipeline(name='add-pipeline') def my_pipeline( a: int = 2, b: int = 5, diff --git a/sdk/python/test_data/pipelines/pipeline_with_reused_component.yaml b/sdk/python/test_data/pipelines/pipeline_with_reused_component.yaml index 69d4234620..d709e5d20a 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_reused_component.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_reused_component.yaml @@ -40,7 +40,6 @@ components: parameters: sum: parameterType: NUMBER_INTEGER -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-add: @@ -150,4 +149,4 @@ root: isOptional: true parameterType: NUMBER_INTEGER schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_task_final_status.yaml b/sdk/python/test_data/pipelines/pipeline_with_task_final_status.yaml index 67a1d2ab17..18ae5fa44d 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_task_final_status.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_task_final_status.yaml @@ -68,15 +68,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -99,15 +101,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -126,15 +130,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.17'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -180,4 +186,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.17 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_task_final_status_yaml.yaml b/sdk/python/test_data/pipelines/pipeline_with_task_final_status_yaml.yaml index 1bfddfd250..5065d84754 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_task_final_status_yaml.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_task_final_status_yaml.yaml @@ -92,4 +92,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.17 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pipeline_with_task_using_ignore_upstream_failure.yaml b/sdk/python/test_data/pipelines/pipeline_with_task_using_ignore_upstream_failure.yaml index d6754e2a11..0ab5a9632e 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_task_using_ignore_upstream_failure.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_task_using_ignore_upstream_failure.yaml @@ -35,15 +35,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -62,15 +64,17 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.0.0-beta.16'\ - \ && \"$0\" \"$@\"\n" + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" - sh - -ec - 'program_path=$(mktemp -d) + printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ @@ -117,4 +121,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.1.3 diff --git a/sdk/python/test_data/pipelines/pipeline_with_various_io_types.py b/sdk/python/test_data/pipelines/pipeline_with_various_io_types.py index 96fdc5b329..934921ee76 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_various_io_types.py +++ b/sdk/python/test_data/pipelines/pipeline_with_various_io_types.py @@ -80,7 +80,7 @@ """) -@dsl.pipeline(name='pipeline-with-various-types', pipeline_root='dummy_root') +@dsl.pipeline(name='pipeline-with-various-types') def my_pipeline(input1: str, input3: Input[Artifact], input4: str = ''): component_1 = component_op_1( input_1=input1, diff --git a/sdk/python/test_data/pipelines/pipeline_with_various_io_types.yaml b/sdk/python/test_data/pipelines/pipeline_with_various_io_types.yaml index 44314129c6..2aae338e84 100644 --- a/sdk/python/test_data/pipelines/pipeline_with_various_io_types.yaml +++ b/sdk/python/test_data/pipelines/pipeline_with_various_io_types.yaml @@ -96,7 +96,6 @@ components: parameters: output_1: parameterType: NUMBER_INTEGER -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-downstream: @@ -213,4 +212,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/pythonic_artifact_with_single_return.py b/sdk/python/test_data/pipelines/pythonic_artifact_with_single_return.py new file mode 100644 index 0000000000..31353e852b --- /dev/null +++ b/sdk/python/test_data/pipelines/pythonic_artifact_with_single_return.py @@ -0,0 +1,58 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from kfp import dsl +from kfp.dsl import Dataset +from kfp.dsl import Model + + +@dsl.component(packages_to_install=['dill==0.3.7']) +def make_language_model(text_dataset: Dataset) -> Model: + # dill allows pickling objects belonging to a function's local namespace + import dill + + with open(text_dataset.path) as f: + text = f.read() + + # insert train on text here # + + def dummy_model(x: str) -> str: + return x + + model = Model( + uri=dsl.get_uri(suffix='model'), + metadata={'data': text_dataset.name}, + ) + + with open(model.path, 'wb') as f: + dill.dump(dummy_model, f) + + return model + + +@dsl.pipeline +def make_language_model_pipeline() -> Model: + importer = dsl.importer( + artifact_uri='gs://ml-pipeline-playground/shakespeare1.txt', + artifact_class=Dataset, + reimport=False, + metadata={'key': 'value'}) + return make_language_model(text_dataset=importer.output).output + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=make_language_model_pipeline, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/pythonic_artifact_with_single_return.yaml b/sdk/python/test_data/pipelines/pythonic_artifact_with_single_return.yaml new file mode 100644 index 0000000000..d010d50a6b --- /dev/null +++ b/sdk/python/test_data/pipelines/pythonic_artifact_with_single_return.yaml @@ -0,0 +1,123 @@ +# PIPELINE DEFINITION +# Name: make-language-model-pipeline +# Outputs: +# Output: system.Model +components: + comp-importer: + executorLabel: exec-importer + inputDefinitions: + parameters: + uri: + parameterType: STRING + outputDefinitions: + artifacts: + artifact: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + comp-make-language-model: + executorLabel: exec-make-language-model + inputDefinitions: + artifacts: + text_dataset: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Model + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-importer: + importer: + artifactUri: + constant: gs://ml-pipeline-playground/shakespeare1.txt + metadata: + key: value + typeSchema: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + exec-make-language-model: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - make_language_model + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.2.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' &&\ + \ python3 -m pip install --quiet --no-warn-script-location 'dill==0.3.7'\ + \ && \"$0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef make_language_model(text_dataset: Dataset) -> Model:\n # dill\ + \ allows pickling objects belonging to a function's local namespace\n \ + \ import dill\n\n with open(text_dataset.path) as f:\n text =\ + \ f.read()\n\n # insert train on text here #\n\n def dummy_model(x:\ + \ str) -> str:\n return x\n\n model = Model(\n uri=dsl.get_uri(suffix='model'),\n\ + \ metadata={'data': text_dataset.name},\n )\n\n with open(model.path,\ + \ 'wb') as f:\n dill.dump(dummy_model, f)\n\n return model\n\n" + image: python:3.7 +pipelineInfo: + name: make-language-model-pipeline +root: + dag: + outputs: + artifacts: + Output: + artifactSelectors: + - outputArtifactKey: Output + producerSubtask: make-language-model + tasks: + importer: + cachingOptions: + enableCache: true + componentRef: + name: comp-importer + inputs: + parameters: + uri: + runtimeValue: + constant: gs://ml-pipeline-playground/shakespeare1.txt + taskInfo: + name: importer + make-language-model: + cachingOptions: + enableCache: true + componentRef: + name: comp-make-language-model + dependentTasks: + - importer + inputs: + artifacts: + text_dataset: + taskOutputArtifact: + outputArtifactKey: artifact + producerTask: importer + taskInfo: + name: make-language-model + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Model + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.2.0 diff --git a/sdk/python/test_data/pipelines/pythonic_artifacts_with_list_of_artifacts.py b/sdk/python/test_data/pipelines/pythonic_artifacts_with_list_of_artifacts.py new file mode 100644 index 0000000000..899bc483df --- /dev/null +++ b/sdk/python/test_data/pipelines/pythonic_artifacts_with_list_of_artifacts.py @@ -0,0 +1,52 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +from kfp import dsl +from kfp.dsl import Dataset + + +@dsl.component +def make_dataset(text: str) -> Dataset: + dataset = Dataset(uri=dsl.get_uri(), metadata={'length': len(text)}) + with open(dataset.path, 'w') as f: + f.write(text) + return dataset + + +@dsl.component +def join_datasets(datasets: List[Dataset]) -> Dataset: + texts = [] + for dataset in datasets: + with open(dataset.path, 'r') as f: + texts.append(f.read()) + + return ''.join(texts) + + +@dsl.pipeline +def make_and_join_datasets( + texts: List[str] = ['Hello', ',', ' ', 'world!']) -> Dataset: + with dsl.ParallelFor(texts) as text: + t1 = make_dataset(text=text) + + return join_datasets(datasets=dsl.Collected(t1.output)).output + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=make_and_join_datasets, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/pythonic_artifacts_with_list_of_artifacts.yaml b/sdk/python/test_data/pipelines/pythonic_artifacts_with_list_of_artifacts.yaml new file mode 100644 index 0000000000..53b5fb17f8 --- /dev/null +++ b/sdk/python/test_data/pipelines/pythonic_artifacts_with_list_of_artifacts.yaml @@ -0,0 +1,187 @@ +# PIPELINE DEFINITION +# Name: make-and-join-datasets +# Inputs: +# texts: list [Default: ['Hello', ',', ' ', 'world!']] +# Outputs: +# Output: system.Dataset +components: + comp-for-loop-1: + dag: + outputs: + artifacts: + pipelinechannel--make-dataset-Output: + artifactSelectors: + - outputArtifactKey: Output + producerSubtask: make-dataset + tasks: + make-dataset: + cachingOptions: + enableCache: true + componentRef: + name: comp-make-dataset + inputs: + parameters: + text: + componentInputParameter: pipelinechannel--texts-loop-item + taskInfo: + name: make-dataset + inputDefinitions: + parameters: + pipelinechannel--texts: + parameterType: LIST + pipelinechannel--texts-loop-item: + parameterType: STRING + outputDefinitions: + artifacts: + pipelinechannel--make-dataset-Output: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + isArtifactList: true + comp-join-datasets: + executorLabel: exec-join-datasets + inputDefinitions: + artifacts: + datasets: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + isArtifactList: true + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + comp-make-dataset: + executorLabel: exec-make-dataset + inputDefinitions: + parameters: + text: + parameterType: STRING + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-join-datasets: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - join_datasets + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.2.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef join_datasets(datasets: List[Dataset]) -> Dataset:\n texts\ + \ = []\n for dataset in datasets:\n with open(dataset.path, 'r')\ + \ as f:\n texts.append(f.read())\n\n return ''.join(texts)\n\ + \n" + image: python:3.7 + exec-make-dataset: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - make_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.2.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef make_dataset(text: str) -> Dataset:\n dataset = Dataset(uri=dsl.get_uri(),\ + \ metadata={'length': len(text)})\n with open(dataset.path, 'w') as f:\n\ + \ f.write(text)\n return dataset\n\n" + image: python:3.7 +pipelineInfo: + name: make-and-join-datasets +root: + dag: + outputs: + artifacts: + Output: + artifactSelectors: + - outputArtifactKey: Output + producerSubtask: join-datasets + tasks: + for-loop-1: + componentRef: + name: comp-for-loop-1 + inputs: + parameters: + pipelinechannel--texts: + componentInputParameter: texts + parameterIterator: + itemInput: pipelinechannel--texts-loop-item + items: + inputParameter: pipelinechannel--texts + taskInfo: + name: for-loop-1 + join-datasets: + cachingOptions: + enableCache: true + componentRef: + name: comp-join-datasets + dependentTasks: + - for-loop-1 + inputs: + artifacts: + datasets: + taskOutputArtifact: + outputArtifactKey: pipelinechannel--make-dataset-Output + producerTask: for-loop-1 + taskInfo: + name: join-datasets + inputDefinitions: + parameters: + texts: + defaultValue: + - Hello + - ',' + - ' ' + - world! + isOptional: true + parameterType: LIST + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.2.0 diff --git a/sdk/python/test_data/pipelines/pythonic_artifacts_with_multiple_returns.py b/sdk/python/test_data/pipelines/pythonic_artifacts_with_multiple_returns.py new file mode 100644 index 0000000000..845ebb7472 --- /dev/null +++ b/sdk/python/test_data/pipelines/pythonic_artifacts_with_multiple_returns.py @@ -0,0 +1,93 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import NamedTuple + +from kfp import dsl +from kfp.dsl import Artifact +from kfp.dsl import Dataset + + +@dsl.component +def dataset_splitter( + in_dataset: Dataset +) -> NamedTuple( + 'outputs', + dataset1=Dataset, + dataset2=Dataset, +): + + with open(in_dataset.path) as f: + in_data = f.read() + + out_data1, out_data2 = in_data[:len(in_data) // 2], in_data[len(in_data) // + 2:] + + dataset1 = Dataset( + uri=dsl.get_uri(suffix='dataset1'), + metadata={'original_data': in_dataset.name}, + ) + with open(dataset1.path, 'w') as f: + f.write(out_data1) + + dataset2 = Dataset( + uri=dsl.get_uri(suffix='dataset2'), + metadata={'original_data': in_dataset.name}, + ) + with open(dataset2.path, 'w') as f: + f.write(out_data2) + + outputs = NamedTuple( + 'outputs', + dataset1=Dataset, + dataset2=Dataset, + ) + return outputs(dataset1=dataset1, dataset2=dataset2) + + +outputs = NamedTuple( + 'outputs', + dataset1=Dataset, + dataset2=Dataset, +) + + +@dsl.pipeline +def splitter_pipeline(in_dataset: Dataset) -> outputs: + task = dataset_splitter(in_dataset=in_dataset) + return outputs( + task.outputs['dataset1'], + task.outputs['dataset1'], + ) + + +@dsl.component +def make_dataset() -> Artifact: + artifact = Artifact(uri=dsl.get_uri('dataset')) + with open(artifact.path, 'w') as f: + f.write('Hello, world') + return artifact + + +@dsl.pipeline +def split_datasets_and_return_first() -> Dataset: + t1 = make_dataset() + return splitter_pipeline(in_dataset=t1.output).outputs['dataset1'] + + +if __name__ == '__main__': + from kfp import compiler + compiler.Compiler().compile( + pipeline_func=split_datasets_and_return_first, + package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/python/test_data/pipelines/pythonic_artifacts_with_multiple_returns.yaml b/sdk/python/test_data/pipelines/pythonic_artifacts_with_multiple_returns.yaml new file mode 100644 index 0000000000..2f655b097c --- /dev/null +++ b/sdk/python/test_data/pipelines/pythonic_artifacts_with_multiple_returns.yaml @@ -0,0 +1,184 @@ +# PIPELINE DEFINITION +# Name: split-datasets-and-return-first +# Outputs: +# Output: system.Dataset +components: + comp-dataset-splitter: + executorLabel: exec-dataset-splitter + inputDefinitions: + artifacts: + in_dataset: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + outputDefinitions: + artifacts: + dataset1: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + dataset2: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + comp-make-dataset: + executorLabel: exec-make-dataset + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-splitter-pipeline: + dag: + outputs: + artifacts: + dataset1: + artifactSelectors: + - outputArtifactKey: dataset1 + producerSubtask: dataset-splitter + dataset2: + artifactSelectors: + - outputArtifactKey: dataset1 + producerSubtask: dataset-splitter + tasks: + dataset-splitter: + cachingOptions: + enableCache: true + componentRef: + name: comp-dataset-splitter + inputs: + artifacts: + in_dataset: + componentInputArtifact: in_dataset + taskInfo: + name: dataset-splitter + inputDefinitions: + artifacts: + in_dataset: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + outputDefinitions: + artifacts: + dataset1: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 + dataset2: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-dataset-splitter: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - dataset_splitter + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.2.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef dataset_splitter(\n in_dataset: Dataset\n) -> NamedTuple(\n\ + \ 'outputs',\n dataset1=Dataset,\n dataset2=Dataset,\n\ + ):\n\n with open(in_dataset.path) as f:\n in_data = f.read()\n\ + \n out_data1, out_data2 = in_data[:len(in_data) // 2], in_data[len(in_data)\ + \ //\n 2:]\n\ + \n dataset1 = Dataset(\n uri=dsl.get_uri(suffix='dataset1'),\n\ + \ metadata={'original_data': in_dataset.name},\n )\n with open(dataset1.path,\ + \ 'w') as f:\n f.write(out_data1)\n\n dataset2 = Dataset(\n \ + \ uri=dsl.get_uri(suffix='dataset2'),\n metadata={'original_data':\ + \ in_dataset.name},\n )\n with open(dataset2.path, 'w') as f:\n \ + \ f.write(out_data2)\n\n outputs = NamedTuple(\n 'outputs',\n\ + \ dataset1=Dataset,\n dataset2=Dataset,\n )\n return\ + \ outputs(dataset1=dataset1, dataset2=dataset2)\n\n" + image: python:3.7 + exec-make-dataset: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - make_dataset + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.2.0'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef make_dataset() -> Artifact:\n artifact = Artifact(uri=dsl.get_uri('dataset'))\n\ + \ with open(artifact.path, 'w') as f:\n f.write('Hello, world')\n\ + \ return artifact\n\n" + image: python:3.7 +pipelineInfo: + name: split-datasets-and-return-first +root: + dag: + outputs: + artifacts: + Output: + artifactSelectors: + - outputArtifactKey: dataset1 + producerSubtask: splitter-pipeline + tasks: + make-dataset: + cachingOptions: + enableCache: true + componentRef: + name: comp-make-dataset + taskInfo: + name: make-dataset + splitter-pipeline: + cachingOptions: + enableCache: true + componentRef: + name: comp-splitter-pipeline + dependentTasks: + - make-dataset + inputs: + artifacts: + in_dataset: + taskOutputArtifact: + outputArtifactKey: Output + producerTask: make-dataset + taskInfo: + name: splitter-pipeline + outputDefinitions: + artifacts: + Output: + artifactType: + schemaTitle: system.Dataset + schemaVersion: 0.0.1 +schemaVersion: 2.1.0 +sdkVersion: kfp-2.2.0 diff --git a/sdk/python/test_data/pipelines/two_step_pipeline.py b/sdk/python/test_data/pipelines/two_step_pipeline.py index 68e3bd701c..16869e2922 100644 --- a/sdk/python/test_data/pipelines/two_step_pipeline.py +++ b/sdk/python/test_data/pipelines/two_step_pipeline.py @@ -52,7 +52,7 @@ """) -@dsl.pipeline(name='simple-two-step-pipeline', pipeline_root='dummy_root') +@dsl.pipeline(name='simple-two-step-pipeline') def my_pipeline(text: str = 'Hello world!'): component_1 = component_op_1(text=text).set_display_name('Producer') component_2 = component_op_2( diff --git a/sdk/python/test_data/pipelines/two_step_pipeline.yaml b/sdk/python/test_data/pipelines/two_step_pipeline.yaml index 0cb2907ca1..b8ebfdd969 100644 --- a/sdk/python/test_data/pipelines/two_step_pipeline.yaml +++ b/sdk/python/test_data/pipelines/two_step_pipeline.yaml @@ -23,7 +23,6 @@ components: artifactType: schemaTitle: system.Artifact schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-read-from-gcs: @@ -89,4 +88,4 @@ root: isOptional: true parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/two_step_pipeline_containerized.yaml b/sdk/python/test_data/pipelines/two_step_pipeline_containerized.yaml index 7f581b4c5c..1ddd43a1c5 100644 --- a/sdk/python/test_data/pipelines/two_step_pipeline_containerized.yaml +++ b/sdk/python/test_data/pipelines/two_step_pipeline_containerized.yaml @@ -78,4 +78,4 @@ root: text: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/pipelines/xgboost_sample_pipeline.py b/sdk/python/test_data/pipelines/xgboost_sample_pipeline.py index 8b51fc0280..cb40d4905d 100644 --- a/sdk/python/test_data/pipelines/xgboost_sample_pipeline.py +++ b/sdk/python/test_data/pipelines/xgboost_sample_pipeline.py @@ -36,7 +36,7 @@ ) -@dsl.pipeline(name='xgboost-sample-pipeline', pipeline_root='dummy_root') +@dsl.pipeline(name='xgboost-sample-pipeline') def xgboost_pipeline(): training_data_csv = chicago_taxi_dataset_op( where='trip_start_timestamp >= "2019-01-01" AND trip_start_timestamp < "2019-02-01"', diff --git a/sdk/python/test_data/pipelines/xgboost_sample_pipeline.yaml b/sdk/python/test_data/pipelines/xgboost_sample_pipeline.yaml index b9ba567619..6555229532 100644 --- a/sdk/python/test_data/pipelines/xgboost_sample_pipeline.yaml +++ b/sdk/python/test_data/pipelines/xgboost_sample_pipeline.yaml @@ -237,7 +237,6 @@ components: artifactType: schemaTitle: system.Artifact schemaVersion: 0.0.1 -defaultPipelineRoot: dummy_root deploymentSpec: executors: exec-chicago-taxi-trips-dataset: @@ -924,4 +923,4 @@ root: taskInfo: name: xgboost-train-2 schemaVersion: 2.1.0 -sdkVersion: kfp-2.0.0-beta.16 +sdkVersion: kfp-2.0.0-rc.2 diff --git a/sdk/python/test_data/test_data_config.yaml b/sdk/python/test_data/test_data_config.yaml index 87958e130a..7421c83303 100644 --- a/sdk/python/test_data/test_data_config.yaml +++ b/sdk/python/test_data/test_data_config.yaml @@ -168,6 +168,27 @@ pipelines: - module: pipeline_with_metadata_fields name: dataset_concatenator execute: false + - module: if_else_with_oneof_artifacts + name: outer_pipeline + execute: false + - module: if_elif_else_complex + name: lucky_number_pipeline + execute: false + - module: if_else_with_oneof_parameters + name: flip_coin_pipeline + execute: false + - module: if_elif_else_with_oneof_parameters + name: outer_pipeline + execute: false + - module: pythonic_artifact_with_single_return + name: make_language_model_pipeline + execute: false + - module: pythonic_artifacts_with_multiple_returns + name: split_datasets_and_return_first + execute: false + - module: pythonic_artifacts_with_list_of_artifacts + name: make_and_join_datasets + execute: false components: test_data_dir: sdk/python/test_data/components read: true @@ -230,9 +251,13 @@ components: execute: false - module: component_with_metadata_fields name: dataset_joiner + execute: false - module: component_with_task_final_status name: exit_comp execute: false + - module: containerized_python_component + name: concat_message + execute: false v1_components: test_data_dir: sdk/python/test_data/v1_component_yaml read: true diff --git a/sdk/runtime_tests/execute_commands_args_test.py b/sdk/runtime_tests/execute_commands_args_test.py new file mode 100644 index 0000000000..42b7672b32 --- /dev/null +++ b/sdk/runtime_tests/execute_commands_args_test.py @@ -0,0 +1,163 @@ +# Copyright 2023 The Kubeflow Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import dataclasses +import json +import os +import re +import shutil +import subprocess +import tempfile +from typing import Any, Dict + +from absl.testing import parameterized +import yaml + +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'test_data') + + +@dataclasses.dataclass +class RuntimeTestConfig: + pipeline_file_relpath: str + executor_name: str + executor_input: Dict[str, Any] + + +TEST_CONFIGS = [ + RuntimeTestConfig( + pipeline_file_relpath=os.path.join( + TEST_DATA_DIR, 'pipeline_with_task_final_status.yaml'), + executor_name='exec-print-op', + executor_input={ + 'inputs': { + 'parameterValues': { + 'message': 'Hello World!' + }, + 'parameters': { + 'message': { + 'stringValue': 'Hello World!' + } + } + }, + 'outputs': { + 'outputFile': + '/gcs/cjmccarthy-kfp-default-bucket/271009669852/pipeline-with-task-final-status-07-14-2023-18-50-32/print-op_-9063136771365142528/executor_output.json' + } + }, + ), + RuntimeTestConfig( + pipeline_file_relpath=os.path.join( + TEST_DATA_DIR, 'pipeline_with_task_final_status.yaml'), + executor_name='exec-exit-op', + executor_input={ + 'inputs': { + 'parameterValues': { + 'status': { + 'error': { + 'code': + 9, + 'message': + 'The DAG failed because some tasks failed. The failed tasks are: [print-op, fail-op].' + }, + 'pipelineJobResourceName': + 'projects/271009669852/locations/us-central1/pipelineJobs/pipeline-with-task-final-status-07-14-2023-19-07-11', + 'pipelineTaskName': + 'my-pipeline', + 'state': + 'FAILED' + }, + 'user_input': 'Hello World!' + }, + 'parameters': { + 'status': { + 'stringValue': + "{\"error\":{\"code\":9,\"message\":\"The DAG failed because some tasks failed. The failed tasks are: [print-op, fail-op].\"},\"pipelineJobResourceName\":\"projects/271009669852/locations/us-central1/pipelineJobs/pipeline-with-task-final-status-07-14-2023-19-07-11\",\"pipelineTaskName\":\"my-pipeline\",\"state\":\"FAILED\"}" + }, + 'user_input': { + 'stringValue': 'Hello World!' + } + } + }, + 'outputs': { + 'outputFile': + '/gcs/cjmccarthy-kfp-default-bucket/271009669852/pipeline-with-task-final-status-07-14-2023-19-07-11/exit-op_-6100894116462198784/executor_output.json' + } + }, + ) +] + +PULL_NUMBER = None + + +def run_commands_and_args( + config: RuntimeTestConfig, + temp_dir: str, +) -> subprocess.CompletedProcess: + with open(config.pipeline_file_relpath) as f: + pipline_spec_dict = yaml.safe_load(f) + container = pipline_spec_dict['deploymentSpec']['executors'][ + config.executor_name]['container'] + + command_and_args = container['command'] + container['args'] + # https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + # pip install from source in a container via a subprocess causes many + # permissions issue + # resolving by modifying the commands/args changes the commands/args + # so much that it renders the test less valuable, since the + # commands/args resemble the true runtime commands/args less well + # prefer the less invasive approach of installing from a PR + global PULL_NUMBER + if PULL_NUMBER is None: + if 'PULL_NUMBER' in os.environ: + PULL_NUMBER = os.environ['PULL_NUMBER'] + else: + PULL_NUMBER = input( + "Please provide the PR number for the kubeflow/pipelines PR that contains the changes you'd like to test:" + ) + + kfp_package_path = f'git+https://github.com/kubeflow/pipelines.git@refs/pull/{PULL_NUMBER}/merge#subdirectory=sdk/python' + command_and_args = [ + re.sub(r"'kfp==(\d+).(\d+).(\d+)(-[a-z]+.\d+)?'", kfp_package_path, + cmd) for cmd in command_and_args + ] + executor_input_json = json.dumps(config.executor_input).replace( + '/gcs/', temp_dir) + command_and_args = [ + v.replace('{{$}}', executor_input_json) for v in command_and_args + ] + + return subprocess.run( + command_and_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +class TestRuntime(parameterized.TestCase): + + @classmethod + def setUp(cls): + cls.temp_dir = tempfile.mkdtemp() + + @classmethod + def tearDown(cls): + shutil.rmtree(cls.temp_dir) + + @parameterized.parameters(TEST_CONFIGS) + def test(self, config: RuntimeTestConfig): + process = run_commands_and_args( + config=config, + temp_dir=self.temp_dir, + ) + self.assertEqual(process.returncode, 0, process.stderr) diff --git a/samples/core/exit_handler/exit_handler_v2.py b/sdk/runtime_tests/test_data/pipeline_with_task_final_status.py similarity index 55% rename from samples/core/exit_handler/exit_handler_v2.py rename to sdk/runtime_tests/test_data/pipeline_with_task_final_status.py index a02122ab4e..27d418a333 100644 --- a/samples/core/exit_handler/exit_handler_v2.py +++ b/sdk/runtime_tests/test_data/pipeline_with_task_final_status.py @@ -1,4 +1,4 @@ -# Copyright 2021 The Kubeflow Authors +# Copyright 2022 The Kubeflow Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,25 +11,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Pipeline using ExitHandler.""" +"""Pipeline using ExitHandler with PipelineTaskFinalStatus.""" -import os -from kfp import dsl from kfp import compiler +from kfp import dsl from kfp.dsl import component +from kfp.dsl import PipelineTaskFinalStatus -# In tests, we install a KFP package from the PR under test. Users should not -# normally need to specify `kfp_package_path` in their component definitions. -_KFP_PACKAGE_PATH = os.getenv('KFP_PACKAGE_PATH') +@component +def exit_op(user_input: str, status: PipelineTaskFinalStatus): + """Checks pipeline run status.""" + print('Pipeline status: ', status.state) + print('Job resource name: ', status.pipeline_job_resource_name) + print('Pipeline task name: ', status.pipeline_task_name) + print('Error code: ', status.error_code) + print('Error message: ', status.error_message) -@component(kfp_package_path=_KFP_PACKAGE_PATH) + +@component def print_op(message: str): """Prints a message.""" print(message) -@component(kfp_package_path=_KFP_PACKAGE_PATH) +@component def fail_op(message: str): """Fails.""" import sys @@ -37,17 +43,16 @@ def fail_op(message: str): sys.exit(1) -@dsl.pipeline(name='pipeline-with-exit-handler') -def pipeline_exit_handler(message: str = 'Hello World!'): - - exit_task = print_op(message='Exit handler has worked!') +@dsl.pipeline(name='pipeline-with-task-final-status') +def my_pipeline(message: str = 'Hello World!'): + exit_task = exit_op(user_input=message) - with dsl.ExitHandler(exit_task): + with dsl.ExitHandler(exit_task, name='my-pipeline'): print_op(message=message) fail_op(message='Task failed.') if __name__ == '__main__': compiler.Compiler().compile( - pipeline_func=pipeline_exit_handler, + pipeline_func=my_pipeline, package_path=__file__.replace('.py', '.yaml')) diff --git a/sdk/runtime_tests/test_data/pipeline_with_task_final_status.yaml b/sdk/runtime_tests/test_data/pipeline_with_task_final_status.yaml new file mode 100644 index 0000000000..18ae5fa44d --- /dev/null +++ b/sdk/runtime_tests/test_data/pipeline_with_task_final_status.yaml @@ -0,0 +1,189 @@ +# PIPELINE DEFINITION +# Name: pipeline-with-task-final-status +# Inputs: +# message: str [Default: 'Hello World!'] +components: + comp-exit-handler-1: + dag: + tasks: + fail-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-fail-op + inputs: + parameters: + message: + runtimeValue: + constant: Task failed. + taskInfo: + name: fail-op + print-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-print-op + inputs: + parameters: + message: + componentInputParameter: pipelinechannel--message + taskInfo: + name: print-op + inputDefinitions: + parameters: + pipelinechannel--message: + parameterType: STRING + comp-exit-op: + executorLabel: exec-exit-op + inputDefinitions: + parameters: + status: + isOptional: true + parameterType: TASK_FINAL_STATUS + user_input: + parameterType: STRING + comp-fail-op: + executorLabel: exec-fail-op + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-print-op: + executorLabel: exec-print-op + inputDefinitions: + parameters: + message: + parameterType: STRING +deploymentSpec: + executors: + exec-exit-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - exit_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef exit_op(user_input: str, status: PipelineTaskFinalStatus):\n\ + \ \"\"\"Checks pipeline run status.\"\"\"\n print('Pipeline status:\ + \ ', status.state)\n print('Job resource name: ', status.pipeline_job_resource_name)\n\ + \ print('Pipeline task name: ', status.pipeline_task_name)\n print('Error\ + \ code: ', status.error_code)\n print('Error message: ', status.error_message)\n\ + \n" + image: python:3.7 + exec-fail-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - fail_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef fail_op(message: str):\n \"\"\"Fails.\"\"\"\n import sys\n\ + \ print(message)\n sys.exit(1)\n\n" + image: python:3.7 + exec-print-op: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - print_op + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.1.3'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef print_op(message: str):\n \"\"\"Prints a message.\"\"\"\n\ + \ print(message)\n\n" + image: python:3.7 +pipelineInfo: + name: pipeline-with-task-final-status +root: + dag: + tasks: + exit-handler-1: + componentRef: + name: comp-exit-handler-1 + inputs: + parameters: + pipelinechannel--message: + componentInputParameter: message + taskInfo: + name: my-pipeline + exit-op: + cachingOptions: + enableCache: true + componentRef: + name: comp-exit-op + dependentTasks: + - exit-handler-1 + inputs: + parameters: + status: + taskFinalStatus: + producerTask: exit-handler-1 + user_input: + componentInputParameter: message + taskInfo: + name: exit-op + triggerPolicy: + strategy: ALL_UPSTREAM_TASKS_COMPLETED + inputDefinitions: + parameters: + message: + defaultValue: Hello World! + isOptional: true + parameterType: STRING +schemaVersion: 2.1.0 +sdkVersion: kfp-2.1.3 diff --git a/test/OWNERS b/test/OWNERS index ce9eb0876f..9c160e6419 100644 --- a/test/OWNERS +++ b/test/OWNERS @@ -2,9 +2,7 @@ approvers: - connor-mccarthy - gkcalat - jlyaoyuli - - Linchin reviewers: - chensun - gkcalat - jlyaoyuli - - Linchin diff --git a/test/deploy-cluster.sh b/test/deploy-cluster.sh index 4949b0e455..39436fadd6 100755 --- a/test/deploy-cluster.sh +++ b/test/deploy-cluster.sh @@ -87,9 +87,14 @@ else SCOPE_ARG="--scopes=storage-rw,cloud-platform" fi # Use regular release channel to keep up with newly created clusters in Google Cloud Marketplace. - # gcloud container clusters create ${TEST_CLUSTER} --release-channel regular ${SCOPE_ARG} ${NODE_POOL_CONFIG_ARG} ${WI_ARG} - # Temporarily use cos as image type until docker dependencies gets removed. reference: https://github.com/kubeflow/pipelines/issues/6696 - gcloud container clusters create ${TEST_CLUSTER} --image-type cos_containerd --release-channel regular ${SCOPE_ARG} ${NODE_POOL_CONFIG_ARG} ${WI_ARG} + # TODO(#9706): Switch back to regular channel once we stop building test images via dind. + # Temporarily use cos as image type until docker dependencies gets removed. + # reference: https://github.com/kubeflow/pipelines/issues/6696 + # Hard-coded GKE to 1.25.10-gke.1200 (the latest 1.25 in STABLE channel). Reference: + # https://github.com/kubeflow/pipelines/issues/9704#issuecomment-1622310358 + # 08/09/2023 update: 1.25.10-gke.1200 no longer supported, use 1.25.10-gke.2100 instead. Reference: + # https://cloud.google.com/kubernetes-engine/docs/release-notes-nochannel#2023-r17_version_updates + gcloud container clusters create ${TEST_CLUSTER} --image-type cos_containerd --release-channel stable --cluster-version 1.25 ${SCOPE_ARG} ${NODE_POOL_CONFIG_ARG} ${WI_ARG} fi gcloud container clusters get-credentials ${TEST_CLUSTER} diff --git a/test/e2e_test_gke_v2.yaml b/test/e2e_test_gke_v2.yaml index ab6a41952f..cdc79cf5ac 100644 --- a/test/e2e_test_gke_v2.yaml +++ b/test/e2e_test_gke_v2.yaml @@ -145,10 +145,7 @@ spec: value: "{{item}}" withItems: - exit_handler - - recursion - sequential - - parallel_join - - volume_ops - name: upgrade-test-preparation inputs: diff --git a/test/frontend-integration-test/helloworld.spec.js b/test/frontend-integration-test/helloworld.spec.js index b409c303cd..785b1a8730 100644 --- a/test/frontend-integration-test/helloworld.spec.js +++ b/test/frontend-integration-test/helloworld.spec.js @@ -33,6 +33,11 @@ async function getValueFromDetailsTable(key) { return rowText.substr(`${key}\n`.length); } +async function clearDefaultInput() { + await browser.keys(['Control', 'a']) + await browser.keys('Backspace'); +} + describe('deploy helloworld sample run', () => { before(async () => { await browser.url('/'); @@ -49,7 +54,9 @@ describe('deploy helloworld sample run', () => { await $('#localPackageBtn').click(); const remoteFilePath = await browser.uploadFile('./helloworld.yaml'); await $('#dropZone input[type="file"]').addValue(remoteFilePath); - await $('#newPipelineName').setValue(pipelineName); + await $('#newPipelineName').click(); + await clearDefaultInput() + await browser.keys(pipelineName) await $('#createNewPipelineOrVersionBtn').click(); await browser.waitUntil(async () => { return new URL(await browser.getUrl()).hash.startsWith('#/pipelines/details'); @@ -99,22 +106,16 @@ describe('deploy helloworld sample run', () => { reverse: true, }); - await browser.keys('Tab'); + await $('#runNameInput').click(); + await clearDefaultInput() await browser.keys(runName); - await browser.keys('Tab'); + await $('#descriptionInput').click(); await browser.keys(runDescription); - - // Skip over "choose experiment" button - await browser.keys('Tab'); - // Skip over service account help button - await browser.keys('Tab'); - // Skip over "service account" textbox - await browser.keys('Tab'); - // Skip over "Run Type" radio button - await browser.keys('Tab'); - - await browser.keys('Tab'); + + // the parameter name is "message" in this testing pipeline + await $('input#newRunPipelineParam0').click(); + await clearDefaultInput() await browser.keys(outputParameterValue); // Deploy @@ -179,6 +180,11 @@ describe('deploy helloworld sample run', () => { ); }); + it('displays run description inputs correctly', async () => { + const descriptionValue = await getValueFromDetailsTable('Description'); + assert.equal(descriptionValue, runDescription, 'run description is not shown correctly'); + }); + it('displays run inputs correctly', async () => { const paramValue = await getValueFromDetailsTable('message'); assert.equal(paramValue, outputParameterValue, 'run message is not shown correctly'); @@ -230,21 +236,14 @@ describe('deploy helloworld sample run', () => { await $('#pipelineSelectorDialog').waitForDisplayed({ timeout: waitTimeout, reverse: true }); + await $('#runNameInput').click(); await browser.keys(runWithoutExperimentName); - await browser.keys('Tab'); + await $('#descriptionInput').click(); await browser.keys(runWithoutExperimentDescription); - - // Skip over "choose experiment" button - await browser.keys('Tab'); - // Skip over service account help button - await browser.keys('Tab'); - // Skip over "service account" textbox - await browser.keys('Tab'); - // Skip over "Run Type" radio button - await browser.keys('Tab'); - - await browser.keys('Tab'); + + await $('input#newRunPipelineParam0').click(); + await clearDefaultInput() await browser.keys(outputParameterValue); // Deploy diff --git a/test/gcpc-tests/run_all_gcpc_modules.py b/test/gcpc-tests/run_all_gcpc_modules.py new file mode 100644 index 0000000000..053fe266f4 --- /dev/null +++ b/test/gcpc-tests/run_all_gcpc_modules.py @@ -0,0 +1,38 @@ +# Copyright 2023 The Kubeflow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test running all GCPC modules.""" +import importlib +import pkgutil +import unittest + + +def run_all_modules(package_name: str) -> None: + package = importlib.import_module(package_name) + for _, module_name, ispkg in pkgutil.walk_packages(package.__path__): + # use dots to avoid false positives on packages with google in name + # and train test split packages + if '.test' in package_name: + continue + if ispkg: + run_all_modules(f'{package_name}.{module_name}') + else: + importlib.import_module(f'{package_name}.{module_name}') + print(f'Successfully ran: {package_name}') + + +class TestRunAllGCPCModules(unittest.TestCase): + + def test_run_all_modules(self): + run_all_modules('google_cloud_pipeline_components.preview') + run_all_modules('google_cloud_pipeline_components.v1') diff --git a/test/kfp-functional-test/README.md b/test/kfp-functional-test/README.md index 8e2414bd07..c08795d322 100644 --- a/test/kfp-functional-test/README.md +++ b/test/kfp-functional-test/README.md @@ -6,8 +6,8 @@ dependencies. To update dependencies: 1. edit [requirements.in](requirements.in) 1. run - ``` - ../../backend/update_requirements.sh requirements.txt + ```bash + pip-compile requirements.in ``` to update and pin the transitive dependencies. @@ -16,42 +16,42 @@ dependencies. To update dependencies: ### Via python 1. run - ``` + ```bash gcloud auth application-default login ``` acquire new user credentials to use for Application Default Credentials. 1. go to the root directory of kubeflow pipelines project, run - ``` + ```bash cd {YOUR_ROOT_DIRECTORY_OF_KUBEFLOW_PIPELINES} python3 ./test/kfp-functional-test/run_kfp_functional_test.py --host "https://$(curl https://raw.githubusercontent.com/kubeflow/testing/master/test-infra/kfp/endpoint)" ``` ### Via docker 1. run - ``` + ```bash gcloud auth application-default login ``` - acquire new user credentials to use for Application Default Credentials. + acquire new user credentials to use for Application Default Credentials. Credentials saved to file with {CREDENTIALS_PATH} similar to: [$HOME/.config/gcloud/application_default_credentials.json] 1. copy the Credentials to the temp folder - ```` + ````bash cp {CREDENTIALS_PATH} /tmp/keys/{FILENAME}.json ``` -1. Provide authentication credentials by setting the environment variable GOOGLE_APPLICATION_CREDENTIALS. +1. Provide authentication credentials by setting the environment variable GOOGLE_APPLICATION_CREDENTIALS. Replace [PATH] with the file path of the JSON file that contains your credentials. - run + run - ``` + ```bash export GOOGLE_APPLICATION_CREDENTIALS="/tmp/keys/{FILENAME}.json" ``` 1. go to the root directory of kubeflow pipelines project and run - ``` + ```bash cd {YOUR_ROOT_DIRECTORY_OF_KUBEFLOW_PIPELINES} docker run -it -v $(pwd):/tmp/src -w /tmp/src -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/keys/{FILENAME}.json \ -v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/keys/{FILENAME}.json:ro \ - python:3.7-slim /tmp/src/test/kfp-functional-test/kfp-functional-test.sh - ``` \ No newline at end of file + python:3.9-slim /tmp/src/test/kfp-functional-test/kfp-functional-test.sh + ``` diff --git a/test/kfp-functional-test/constants.py b/test/kfp-functional-test/constants.py index fd88b6f420..145be20dcc 100644 --- a/test/kfp-functional-test/constants.py +++ b/test/kfp-functional-test/constants.py @@ -14,3 +14,4 @@ # Common test params RUN_TIMEOUT_SECONDS = 1800 +DEFAULT_USER_NAMESPACE = 'kubeflow-user-example-com' diff --git a/test/kfp-functional-test/kfp-functional-test.sh b/test/kfp-functional-test/kfp-functional-test.sh index 4fb7052fd3..2e07f48f44 100755 --- a/test/kfp-functional-test/kfp-functional-test.sh +++ b/test/kfp-functional-test/kfp-functional-test.sh @@ -1,5 +1,5 @@ #!/bin/sh -ex -# Copyright 2020 The Kubeflow Authors +# Copyright 2023 The Kubeflow Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,8 +16,6 @@ apt-get update -y apt --no-install-recommends -y -q install curl source_root="$(pwd)" -# TODO(#4853) Skipping pip 20.3 due to a bad version resolution logic. -python3 -m pip install --upgrade pip!=20.3.* python3 -m pip install -r "${source_root}/test/kfp-functional-test/requirements.txt" HOST="https://$(curl https://raw.githubusercontent.com/kubeflow/testing/master/test-infra/kfp/endpoint)" diff --git a/test/kfp-functional-test/requirements.in b/test/kfp-functional-test/requirements.in index db6f93edcb..6dd00f161e 100644 --- a/test/kfp-functional-test/requirements.in +++ b/test/kfp-functional-test/requirements.in @@ -1 +1 @@ -kfp==1.1.2 +kfp==2.0.1 diff --git a/test/kfp-functional-test/requirements.txt b/test/kfp-functional-test/requirements.txt index f3c8124283..a0cba588a7 100644 --- a/test/kfp-functional-test/requirements.txt +++ b/test/kfp-functional-test/requirements.txt @@ -2,10 +2,8 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file=- - +# pip-compile requirements.in # -attrs==20.3.0 - # via jsonschema cachetools==4.1.1 # via google-auth certifi==2020.12.5 @@ -17,70 +15,70 @@ cffi==1.14.4 # via google-crc32c chardet==3.0.4 # via requests -click==7.1.2 - # via kfp -cloudpickle==1.6.0 - # via kfp -deprecated==1.2.10 +click==8.1.6 # via kfp docstring-parser==0.7.3 # via kfp -google-api-core==1.23.0 - # via google-cloud-core -google-auth==1.23.0 +google-api-core==2.11.1 + # via + # google-cloud-core + # google-cloud-storage + # kfp +google-auth==2.22.0 # via # google-api-core + # google-cloud-core # google-cloud-storage # kfp # kubernetes -google-cloud-core==1.4.4 +google-cloud-core==2.3.3 # via google-cloud-storage -google-cloud-storage==1.32.0 +google-cloud-storage==2.10.0 # via kfp google-crc32c==1.0.0 # via google-resumable-media -google-resumable-media==1.1.0 +google-resumable-media==2.5.0 # via google-cloud-storage -googleapis-common-protos==1.52.0 +googleapis-common-protos==1.60.0 # via google-api-core idna==2.10 # via requests -jsonschema==3.2.0 +kfp-pipeline-spec==0.2.2 # via kfp -kfp==1.1.2 - # via -r requirements.in -kfp-pipeline-spec==0.1.3.1 - # via kfp -kfp-server-api==1.1.2rc1 +kfp-server-api==2.0.0 # via kfp +kfp==2.0.1 + # via -r requirements.in kubernetes==11.0.0 # via kfp oauthlib==3.1.0 # via requests-oauthlib -protobuf==3.14.0 +protobuf==3.20.3 # via # google-api-core # googleapis-common-protos + # kfp + # kfp-pipeline-spec +pyasn1-modules==0.2.8 + # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 - # via google-auth pycparser==2.20 # via cffi -pyrsistent==0.17.3 - # via jsonschema python-dateutil==2.8.1 # via # kfp-server-api # kubernetes -pytz==2020.4 - # via google-api-core pyyaml==5.3.1 # via # kfp # kubernetes +requests-oauthlib==1.3.0 + # via kubernetes +requests-toolbelt==0.9.1 + # via kfp requests==2.25.0 # via # google-api-core @@ -88,39 +86,26 @@ requests==2.25.0 # kubernetes # requests-oauthlib # requests-toolbelt -requests-oauthlib==1.3.0 - # via kubernetes -requests-toolbelt==0.9.1 - # via kfp rsa==4.6 # via google-auth six==1.15.0 # via - # google-api-core # google-auth - # google-cloud-core - # google-resumable-media - # jsonschema # kfp-server-api # kubernetes - # protobuf # python-dateutil # websocket-client -strip-hints==0.1.9 - # via kfp tabulate==0.8.7 # via kfp urllib3==1.26.2 # via + # google-auth + # kfp # kfp-server-api # kubernetes # requests websocket-client==0.57.0 # via kubernetes -wheel==0.38.1 - # via strip-hints -wrapt==1.12.1 - # via deprecated # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/test/kfp-functional-test/run_kfp_functional_test.py b/test/kfp-functional-test/run_kfp_functional_test.py index faf1083a9c..00f031f3a4 100644 --- a/test/kfp-functional-test/run_kfp_functional_test.py +++ b/test/kfp-functional-test/run_kfp_functional_test.py @@ -1,4 +1,4 @@ -# Copyright 2020 The Kubeflow Authors +# Copyright 2023 The Kubeflow Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,30 +13,24 @@ # limitations under the License. import argparse +from datetime import datetime import random import string -from datetime import datetime -import kfp.deprecated as kfp -from kfp.deprecated import dsl import constants +import kfp +import kfp.dsl as dsl -def echo_op(): - return dsl.ContainerOp( - name='echo', - image='library/bash:4.4.23', - command=['sh', '-c'], - arguments=['echo "hello world"'] - ) +@dsl.container_component +def say_hello(name: str): + return dsl.ContainerSpec( + image='library/bash:4.4.23', command=['echo'], args=[f'Hello, {name}!']) -@dsl.pipeline( - name='My first pipeline', - description='A hello world pipeline.' -) -def hello_world_pipeline(): - echo_task = echo_op() +@dsl.pipeline(name='My first pipeline', description='A hello pipeline.') +def hello_pipeline(name: str): + say_hello(name=name) # Parsing the input arguments @@ -44,10 +38,8 @@ def parse_arguments(): """Parse command line arguments.""" parser = argparse.ArgumentParser() - parser.add_argument('--host', - type=str, - required=True, - help='The host of kfp.') + parser.add_argument( + '--host', type=str, required=True, help='The host of kfp.') args = parser.parse_args() return args @@ -57,40 +49,44 @@ def main(): ###### Initialization ###### client = kfp.Client(args.host) - print("host is {}".format(args.host)) + print('host is {}'.format(args.host)) ###### Create Experiment ###### - print("Creating experiment") - experiment_name = "kfp-functional-e2e-expriment-" + "".join(random.choices(string.ascii_uppercase + - string.digits, k=5)) - response = client.create_experiment(experiment_name) - experiment_id = response.id - print("Experiment with id {} created".format(experiment_id)) + print('Creating experiment') + experiment_name = 'kfp-functional-e2e-expriment-' + ''.join( + random.choices(string.ascii_uppercase + string.digits, k=5)) + response = client.create_experiment( + experiment_name, namespace=constants.DEFAULT_USER_NAMESPACE) + experiment_id = response.experiment_id + print('Experiment with id {} created'.format(experiment_id)) try: ###### Create Run from Pipeline Func ###### - print("Creating Run from Pipeline Func") - response = client.create_run_from_pipeline_func(hello_world_pipeline, arguments={}, experiment_name=experiment_name) + print('Creating Run from Pipeline Func') + response = client.create_run_from_pipeline_func( + hello_pipeline, + arguments={'name': 'World'}, + experiment_name=experiment_name, + namespace=constants.DEFAULT_USER_NAMESPACE) run_id = response.run_id - print("Run {} created".format(run_id)) + print('Run {} created'.format(run_id)) ###### Monitor Run ###### start_time = datetime.now() - response = client.wait_for_run_completion(run_id, constants.RUN_TIMEOUT_SECONDS) - succ = (response.run.status.lower() == 'succeeded') + response = client.wait_for_run_completion(run_id, + constants.RUN_TIMEOUT_SECONDS) + success = (response.state.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds - if succ: - print("Run succeeded in {} seconds".format(elapsed_time)) + if success: + print('Run succeeded in {} seconds'.format(elapsed_time)) else: print("Run can't complete in {} seconds".format(elapsed_time)) finally: ###### Archive Experiment ###### - print("Archive experiment has a serious performance problem right now, so we temporarily disable it.") - print("TODO(Bobgy): re-enable archiving experiment action after fixing https://github.com/kubeflow/pipelines/issues/6815#issuecomment-955938098") - # print("Archiving experiment") - # client.experiments.archive_experiment(experiment_id) - # print("Archived experiment with id {}".format(experiment_id)) + print('Archiving experiment') + client.archive_experiment(experiment_id) + print('Archived experiment with id {}'.format(experiment_id)) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/test/presubmit-component-yaml.sh b/test/presubmit-component-yaml.sh index 7bc30f8f63..62a4f403ed 100755 --- a/test/presubmit-component-yaml.sh +++ b/test/presubmit-component-yaml.sh @@ -15,10 +15,8 @@ source_root=$(pwd) -pushd $source_root/sdk/python python3 -m pip install --upgrade pip -python3 -m pip install -r requirements.txt -python3 -m pip install . +python3 -m pip install sdk/python # Test loading all component.yaml definitions "$source_root/components/test_load_all_components.sh" diff --git a/test/presubmit-test-kfp-kubernetes-library.sh b/test/presubmit-test-kfp-kubernetes-library.sh index 931b03831e..a8225de5f7 100755 --- a/test/presubmit-test-kfp-kubernetes-library.sh +++ b/test/presubmit-test-kfp-kubernetes-library.sh @@ -18,9 +18,7 @@ source_root=$(pwd) pip install --upgrade pip pip install wheel -## start remove: install from PyPI once platform-specific features are released in the KFP SDK -pip install -e "$source_root/sdk/python" -## end remove +pip install 'kfp>=2.0.0,<3.0.0' # generate Python proto code from source apt-get update -y diff --git a/test/presubmit-test-kfp-runtime-code.sh b/test/presubmit-test-kfp-runtime-code.sh new file mode 100755 index 0000000000..3e1196c647 --- /dev/null +++ b/test/presubmit-test-kfp-runtime-code.sh @@ -0,0 +1,33 @@ +#!/bin/bash -ex +# Copyright 2023 Kubeflow Pipelines contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +source_root=$(pwd) + +pip install --upgrade pip +pip install pyyaml +pip install $(grep 'absl-py==' sdk/python/requirements-dev.txt) + +# precautionarilty uninstall typing-extensions, in case any of the test libs +# installed require this dep. we want to test that the kfp sdk installs it, so +# it cannot be present in the environment prior to test execution. +# we'd rather tests fail to execute (false positive failure) because a test +# lib was missing its dependency on typing-extensions than get a false +# negative from the actual kfp sdk test because typing-extensions was already +# present in the environment. +pip uninstall typing-extensions -y + +# run with unittest because pytest requires typing-extensions +python -m unittest discover -s sdk/runtime_tests -p '*_test.py' diff --git a/test/presubmit-test-run-all-gcpc-modules.sh b/test/presubmit-test-run-all-gcpc-modules.sh new file mode 100755 index 0000000000..81389e128a --- /dev/null +++ b/test/presubmit-test-run-all-gcpc-modules.sh @@ -0,0 +1,23 @@ +#!/bin/bash -ex +# Copyright 2023 Kubeflow Pipelines contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source_root=$(pwd) + +pip install --upgrade pip +pip install $source_root/sdk/python +pip install components/google-cloud +pip install $(grep 'pytest==' sdk/python/requirements-dev.txt) + +pytest test/gcpc-tests/run_all_gcpc_modules.py diff --git a/test/presubmit-test-sdk-upgrade.sh b/test/presubmit-test-sdk-upgrade.sh new file mode 100755 index 0000000000..eb93ded09f --- /dev/null +++ b/test/presubmit-test-sdk-upgrade.sh @@ -0,0 +1,31 @@ +#!/bin/bash -ex +# Copyright 2023 Kubeflow Pipelines contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +python3 -m pip install --upgrade pip + +python3 -m pip install kfp +LATEST_KFP_SDK_RELEASE=$(python3 -m pip show kfp | grep "Version:" | awk '{print $2}' | awk '{$1=$1};1') +echo "Installed latest KFP SDK version: $LATEST_KFP_SDK_RELEASE" + +# install in normal mode, not editable mode, to emulate typical user upgrade behavior +python3 -m pip install sdk/python +# HEAD will only be different than latest for a release PR +HEAD_KFP_SDK_VERSION=$(python3 -m pip show kfp | grep "Version:" | awk '{print $2}') +echo "Successfully upgraded to KFP SDK version @ HEAD: $HEAD_KFP_SDK_VERSION" + +python3 -c 'import kfp' +echo "Successfully ran 'import kfp' @ HEAD: $HEAD_KFP_SDK_VERSION" diff --git a/test/presubmit-tests-sdk.sh b/test/presubmit-tests-sdk.sh index 97a6c26444..7ce14e8bb5 100755 --- a/test/presubmit-tests-sdk.sh +++ b/test/presubmit-tests-sdk.sh @@ -24,12 +24,10 @@ python3 -m pip install $(grep 'pytest-xdist==' sdk/python/requirements-dev.txt) python3 -m pip install $(grep 'pytest-cov==' sdk/python/requirements-dev.txt) python3 -m pip install --upgrade protobuf -pushd "$source_root/sdk/python" -python3 -m pip install -e . -popd # Changing the current directory to the repo root for correct coverall paths +python3 -m pip install sdk/python # TODO: remove deprecated dependency; then remove --ignore arg -pytest sdk/python/kfp --ignore=sdk/python/kfp/deprecated --cov=kfp +pytest sdk/python/kfp --ignore=sdk/python/kfp/deprecated --cov=kfp set +x # export COVERALLS_REPO_TOKEN=$(gsutil cat gs://ml-pipeline-test-keys/coveralls_repo_token) diff --git a/test/presubmit-tests-tfx.sh b/test/presubmit-tests-tfx.sh index 0136ded812..e26a8f9fa5 100755 --- a/test/presubmit-tests-tfx.sh +++ b/test/presubmit-tests-tfx.sh @@ -30,9 +30,7 @@ pip3 install junit_xml # Using Argo to lint all compiled workflows "${source_root}/test/install-argo-cli.sh" -pushd $source_root/sdk/python -python3 -m pip install -e . -popd # Changing the current directory to the repo root for correct coverall paths +python3 -m pip install sdk/python # Test against TFX # Compile and setup bazel for compiling the protos diff --git a/test/sample-test/Dockerfile b/test/sample-test/Dockerfile index 9f661271ab..ffc3bbd0a8 100644 --- a/test/sample-test/Dockerfile +++ b/test/sample-test/Dockerfile @@ -16,7 +16,7 @@ RUN pip3 install -r /python/src/github.com/kubeflow/pipelines/test/sample-test/r # Install python client, including DSL compiler. # COPY ./sdk/python /sdk/python # RUN pip3 install /sdk/python -RUN pip3 install kfp~=1.8 +RUN pip3 install kfp~=2.0 # Copy sample test and samples source code. COPY ./test/sample-test /python/src/github.com/kubeflow/pipelines/test/sample-test diff --git a/test/sample-test/run_sample_test.py b/test/sample-test/run_sample_test.py index 2ec1765217..1a77d2fd05 100644 --- a/test/sample-test/run_sample_test.py +++ b/test/sample-test/run_sample_test.py @@ -12,40 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +from datetime import datetime import os import tarfile import time + +from constants import CONFIG_DIR +from constants import DEFAULT_CONFIG +from constants import SCHEMA_CONFIG +import kfp +from kfp import Client import utils import yamale import yaml -from datetime import datetime -from kfp import Client -from constants import CONFIG_DIR, DEFAULT_CONFIG, SCHEMA_CONFIG class PySampleChecker(object): - def __init__( - self, - testname, - input, - output, - result, - experiment_name, - host, - namespace='kubeflow' - ): + def __init__(self, + testname, + input, + output, + result, + experiment_name, + host, + namespace='kubeflow'): """Util class for checking python sample test running results. - :param testname: test name. - :param input: The path of a pipeline file that will be submitted. - :param output: The path of the test output. - :param result: The path of the test result that will be exported. - :param host: The hostname of KFP API endpoint. - :param namespace: namespace of the deployed pipeline system. Default: kubeflow - :param experiment_name: Name of the experiment to monitor - """ + :param testname: test name. + :param input: The path of a pipeline file that will be submitted. + :param output: The path of the test output. + :param result: The path of the test result that will be exported. + :param host: The hostname of KFP API endpoint. + :param namespace: namespace of the deployed pipeline system. Default: kubeflow + :param experiment_name: Name of the experiment to monitor + """ self._testname = testname self._experiment_name = experiment_name self._input = input @@ -72,20 +73,18 @@ def run(self): self._client = Client(host=self._host) ###### Check Input File ###### - utils.add_junit_test( - self._test_cases, 'input generated yaml file', - os.path.exists(self._input), 'yaml file is not generated' - ) + utils.add_junit_test(self._test_cases, 'input generated yaml file', + os.path.exists(self._input), + 'yaml file is not generated') if not os.path.exists(self._input): - utils.write_junit_xml( - self._test_name, self._result, self._test_cases - ) + utils.write_junit_xml(self._test_name, self._result, + self._test_cases) print('Error: job not found.') exit(1) ###### Create Experiment ###### response = self._client.create_experiment(self._experiment_name) - self._experiment_id = response.id + self._experiment_id = response.experiment_id utils.add_junit_test(self._test_cases, 'create experiment', True) ###### Create Job ###### @@ -98,8 +97,8 @@ def run(self): raw_args = yaml.safe_load(f) default_config = yamale.make_data(DEFAULT_CONFIG) yamale.validate( - config_schema, default_config - ) # If fails, a ValueError will be raised. + config_schema, + default_config) # If fails, a ValueError will be raised. except yaml.YAMLError as yamlerr: raise RuntimeError('Illegal default config:{}'.format(yamlerr)) except OSError as ose: @@ -109,25 +108,21 @@ def run(self): self._run_pipeline = raw_args['run_pipeline'] try: - config_file = os.path.join( - CONFIG_DIR, '%s.config.yaml' % self._testname - ) + config_file = os.path.join(CONFIG_DIR, + '%s.config.yaml' % self._testname) with open(config_file, 'r') as f: raw_args = yaml.safe_load(f) test_config = yamale.make_data(config_file) yamale.validate( - config_schema, test_config - ) # If fails, a ValueError will be raised. + config_schema, + test_config) # If fails, a ValueError will be raised. except yaml.YAMLError as yamlerr: - print( - 'No legit yaml config file found, use default args:{}'. - format(yamlerr) - ) + print('No legit yaml config file found, use default args:{}'.format( + yamlerr)) except OSError as ose: print( - 'Config file with the same name not found, use default args:{}'. - format(ose) - ) + 'Config file with the same name not found, use default args:{}' + .format(ose)) else: if 'arguments' in raw_args.keys() and raw_args['arguments']: self._test_args.update(raw_args['arguments']) @@ -143,17 +138,15 @@ def run(self): if self._testname == 'parameterized_tfx_oss': self._test_args['pipeline-root'] = os.path.join( self._test_args['output'], - 'tfx_taxi_simple_' + kfp.dsl.RUN_ID_PLACEHOLDER - ) + 'tfx_taxi_simple_' + kfp.dsl.RUN_ID_PLACEHOLDER) del self._test_args['output'] # Submit for pipeline running. if self._run_pipeline: - response = self._client.run_pipeline( - self._experiment_id, self._job_name, self._input, - self._test_args - ) - self._run_id = response.id + response = self._client.run_pipeline(self._experiment_id, + self._job_name, self._input, + self._test_args) + self._run_id = response.run_id utils.add_junit_test(self._test_cases, 'create pipeline run', True) def check(self): @@ -163,54 +156,22 @@ def check(self): try: start_time = datetime.now() response = self._client.wait_for_run_completion( - self._run_id, self._test_timeout - ) - succ = (response.run.status.lower() == 'succeeded') + self._run_id, self._test_timeout) + succ = (response.state.lower() == 'succeeded') end_time = datetime.now() elapsed_time = (end_time - start_time).seconds - utils.add_junit_test( - self._test_cases, 'job completion', succ, - 'waiting for job completion failure', elapsed_time - ) + utils.add_junit_test(self._test_cases, 'job completion', succ, + 'waiting for job completion failure', + elapsed_time) finally: - ###### Output Argo Log for Debugging ###### - workflow_json = self._client._get_workflow_json(self._run_id) - workflow_id = workflow_json['metadata']['name'] - print("Argo Workflow Name: ", workflow_id) - argo_log, _ = utils.run_bash_command( - 'argo logs {} -n {}'.format( - workflow_id, self._namespace - ) - ) - print('=========Argo Workflow Log=========') - print(argo_log) + # TODO(chensun): print log for debugging + pass if not succ: - utils.write_junit_xml( - self._test_name, self._result, self._test_cases - ) + utils.write_junit_xml(self._test_name, self._result, + self._test_cases) exit(1) - ###### Validate the results for specific test cases ###### - if self._testname == 'xgboost_training_cm': - # For xgboost sample, check its confusion matrix. - cm_tar_path = './confusion_matrix.tar.gz' - utils.get_artifact_in_minio( - workflow_json, 'confusion-matrix', cm_tar_path, - 'mlpipeline-ui-metadata' - ) - with tarfile.open(cm_tar_path) as tar_handle: - file_handles = tar_handle.getmembers() - assert len(file_handles) == 1 - - with tar_handle.extractfile(file_handles[0]) as f: - cm_data = f.read() - utils.add_junit_test( - self._test_cases, 'confusion matrix format', - (len(cm_data) > 0), - 'the confusion matrix file is empty' - ) - ###### Delete Job ###### #TODO: add deletion when the backend API offers the interface. diff --git a/test/sample-test/sample_test_launcher.py b/test/sample-test/sample_test_launcher.py index f9bd30a27f..7b02556f5c 100644 --- a/test/sample-test/sample_test_launcher.py +++ b/test/sample-test/sample_test_launcher.py @@ -11,250 +11,292 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -This launcher module serves as the entry-point of the sample test image. It -decides which test to trigger based upon the arguments provided. +"""This launcher module serves as the entry-point of the sample test image. + +It decides which test to trigger based upon the arguments provided. """ -import fire import os -import papermill as pm +import pathlib import re import subprocess -import utils -import yamale -import yaml -import kubernetes -from constants import PAPERMILL_ERR_MSG, BASE_DIR, TEST_DIR, SCHEMA_CONFIG, CONFIG_DIR, DEFAULT_CONFIG from check_notebook_results import NoteBookChecker -from kfp.containers._gcs_helper import GCSHelper +from constants import BASE_DIR +from constants import CONFIG_DIR +from constants import DEFAULT_CONFIG +from constants import PAPERMILL_ERR_MSG +from constants import SCHEMA_CONFIG +from constants import TEST_DIR +import fire +import kubernetes +import papermill as pm from run_sample_test import PySampleChecker +import utils +import yamale +import yaml class SampleTest(object): - def __init__(self, test_name, results_gcs_dir, host='', target_image_prefix='', - namespace='kubeflow'): - """Launch a KFP sample_test provided its name. - - :param test_name: name of the corresponding sample test. - :param results_gcs_dir: gs dir to store test result. - :param host: host of KFP API endpoint, default is auto-discovery from inverse-proxy-config. - :param target_image_prefix: prefix of docker image, default is empty. - :param namespace: namespace for kfp, default is kubeflow. - """ - self._test_name = test_name - self._results_gcs_dir = results_gcs_dir - # Capture the first segment after gs:// as the project name. - self._bucket_name = results_gcs_dir.split('/')[2] - self._target_image_prefix = target_image_prefix - self._namespace = namespace - self._host = host - if self._host == '': - try: - # Get inverse proxy hostname from a config map called 'inverse-proxy-config' - # in the same namespace as KFP. + def __init__(self, + test_name, + results_gcs_dir, + host='', + target_image_prefix='', + namespace='kubeflow'): + """Launch a KFP sample_test provided its name. + + :param test_name: name of the corresponding sample test. + :param results_gcs_dir: gs dir to store test result. + :param host: host of KFP API endpoint, default is auto-discovery from inverse-proxy-config. + :param target_image_prefix: prefix of docker image, default is empty. + :param namespace: namespace for kfp, default is kubeflow. + """ + self._test_name = test_name + self._results_gcs_dir = results_gcs_dir + # Capture the first segment after gs:// as the project name. + self._bucket_name = results_gcs_dir.split('/')[2] + self._target_image_prefix = target_image_prefix + self._namespace = namespace + self._host = host + if self._host == '': + try: + # Get inverse proxy hostname from a config map called 'inverse-proxy-config' + # in the same namespace as KFP. + try: + kubernetes.config.load_incluster_config() + except: + kubernetes.config.load_kube_config() + + v1 = kubernetes.client.CoreV1Api() + inverse_proxy_config = v1.read_namespaced_config_map( + name='inverse-proxy-config', namespace=self._namespace) + self._host = inverse_proxy_config.data.get('Hostname') + except Exception as err: + raise RuntimeError( + 'Failed to get inverse proxy hostname') from err + # Keep as comment here, we can also specify host in-cluster as the following, + # but we no longer use it in e2e tests, because we prefer including + # test coverage for inverse proxy. + # self._host = 'ml-pipeline.%s.svc.cluster.local:8888' % self._namespace + print('KFP API host is %s' % self._host) + + self._is_notebook = None + self._work_dir = os.path.join(BASE_DIR, 'samples/core/', + self._test_name) + + self._sample_test_result = 'junit_Sample%sOutput.xml' % self._test_name + self._sample_test_output = self._results_gcs_dir + + def _copy_result(self): + """Copy generated sample test result to gcs, so that Prow can pick + it.""" + + def _upload_gcs_file(local_path: str, gcs_path: str): + from google.cloud import storage + pure_path = pathlib.PurePath(gcs_path) + gcs_bucket = pure_path.parts[1] + gcs_blob = '/'.join(pure_path.parts[2:]) + client = storage.Client() + bucket = client.get_bucket(gcs_bucket) + blob = bucket.blob(gcs_blob) + blob.upload_from_filename(local_path) + + print('Copy the test results to GCS %s/' % self._results_gcs_dir) + + _upload_gcs_file( + self._sample_test_result, + os.path.join(self._results_gcs_dir, self._sample_test_result)) + + def _compile(self): + + os.chdir(self._work_dir) + print('Run the sample tests...') + + # Looking for the entry point of the test. + list_of_files = os.listdir('.') + for file in list_of_files: + m = re.match(self._test_name + '\.[a-zA-Z]+', file) + if m: + file_name, ext_name = os.path.splitext(file) + if self._is_notebook is not None: + raise (RuntimeError( + 'Multiple entry points found under sample: {}'.format( + self._test_name))) + if ext_name == '.py': + self._is_notebook = False + if ext_name == '.ipynb': + self._is_notebook = True + + if self._is_notebook is None: + raise (RuntimeError('No entry point found for sample: {}'.format( + self._test_name))) + + config_schema = yamale.make_schema(SCHEMA_CONFIG) + # Retrieve default config try: - kubernetes.config.load_incluster_config() - except: - kubernetes.config.load_kube_config() - - v1 = kubernetes.client.CoreV1Api() - inverse_proxy_config = v1.read_namespaced_config_map(name='inverse-proxy-config', namespace=self._namespace) - self._host = inverse_proxy_config.data.get('Hostname') - except Exception as err: - raise RuntimeError('Failed to get inverse proxy hostname') from err - # Keep as comment here, we can also specify host in-cluster as the following, - # but we no longer use it in e2e tests, because we prefer including - # test coverage for inverse proxy. - # self._host = 'ml-pipeline.%s.svc.cluster.local:8888' % self._namespace - print('KFP API host is %s' % self._host) - - self._is_notebook = None - self._work_dir = os.path.join(BASE_DIR, 'samples/core/', self._test_name) - - self._sample_test_result = 'junit_Sample%sOutput.xml' % self._test_name - self._sample_test_output = self._results_gcs_dir - - def _copy_result(self): - """ Copy generated sample test result to gcs, so that Prow can pick it. """ - print('Copy the test results to GCS %s/' % self._results_gcs_dir) - - GCSHelper.upload_gcs_file( - self._sample_test_result, - os.path.join(self._results_gcs_dir, self._sample_test_result)) - - def _compile(self): - - os.chdir(self._work_dir) - print('Run the sample tests...') - - # Looking for the entry point of the test. - list_of_files = os.listdir('.') - for file in list_of_files: - m = re.match(self._test_name + '\.[a-zA-Z]+', file) - if m: - file_name, ext_name = os.path.splitext(file) - if self._is_notebook is not None: - raise(RuntimeError('Multiple entry points found under sample: {}'.format(self._test_name))) - if ext_name == '.py': - self._is_notebook = False - if ext_name == '.ipynb': - self._is_notebook = True - - if self._is_notebook is None: - raise(RuntimeError('No entry point found for sample: {}'.format(self._test_name))) - - config_schema = yamale.make_schema(SCHEMA_CONFIG) - # Retrieve default config - try: - with open(DEFAULT_CONFIG, 'r') as f: - raw_args = yaml.safe_load(f) - default_config = yamale.make_data(DEFAULT_CONFIG) - yamale.validate(config_schema, default_config) # If fails, a ValueError will be raised. - except yaml.YAMLError as yamlerr: - raise RuntimeError('Illegal default config:{}'.format(yamlerr)) - except OSError as ose: - raise FileExistsError('Default config not found:{}'.format(ose)) - else: - self._run_pipeline = raw_args['run_pipeline'] - - # For presubmit check, do not do any image injection as for now. - # Notebook samples need to be papermilled first. - if self._is_notebook: - # Parse necessary params from config.yaml - nb_params = {} - try: - config_file = os.path.join(CONFIG_DIR, '%s.config.yaml' % self._test_name) - with open(config_file, 'r') as f: - raw_args = yaml.safe_load(f) - test_config = yamale.make_data(config_file) - yamale.validate(config_schema, test_config) # If fails, a ValueError will be raised. - except yaml.YAMLError as yamlerr: - print('No legit yaml config file found, use default args:{}'.format(yamlerr)) - except OSError as ose: - print('Config file with the same name not found, use default args:{}'.format(ose)) - else: - if 'notebook_params' in raw_args.keys(): - nb_params.update(raw_args['notebook_params']) - if 'output' in raw_args['notebook_params'].keys(): # output is a special param that has to be specified dynamically. - nb_params['output'] = self._sample_test_output - if 'run_pipeline' in raw_args.keys(): - self._run_pipeline = raw_args['run_pipeline'] - - pm.execute_notebook( - input_path='%s.ipynb' % self._test_name, - output_path='%s.ipynb' % self._test_name, - parameters=nb_params, - prepare_only=True - ) - # Convert to python script. - subprocess.call([ - 'jupyter', 'nbconvert', '--to', 'python', '%s.ipynb' % self._test_name - ]) - - else: - subprocess.call(['python3', '%s.py' % self._test_name]) - - def _injection(self): - """Inject images for pipeline components. - This is only valid for coimponent test - """ - pass - - def run_test(self): - self._compile() - self._injection() - - # Overriding the experiment name of pipeline runs - experiment_name = self._test_name + '-test' - os.environ['KF_PIPELINES_OVERRIDE_EXPERIMENT_NAME'] = experiment_name - - if self._is_notebook: - nbchecker = NoteBookChecker(testname=self._test_name, - result=self._sample_test_result, - run_pipeline=self._run_pipeline, - experiment_name=experiment_name, - host=self._host, - ) - nbchecker.run() - os.chdir(TEST_DIR) - nbchecker.check() - else: - os.chdir(TEST_DIR) - input_file = os.path.join(self._work_dir, '%s.py.yaml' % self._test_name) - - pysample_checker = PySampleChecker(testname=self._test_name, - input=input_file, - output=self._sample_test_output, - result=self._sample_test_result, - host=self._host, - namespace=self._namespace, - experiment_name=experiment_name, - ) - pysample_checker.run() - pysample_checker.check() - - self._copy_result() + with open(DEFAULT_CONFIG, 'r') as f: + raw_args = yaml.safe_load(f) + default_config = yamale.make_data(DEFAULT_CONFIG) + yamale.validate( + config_schema, + default_config) # If fails, a ValueError will be raised. + except yaml.YAMLError as yamlerr: + raise RuntimeError('Illegal default config:{}'.format(yamlerr)) + except OSError as ose: + raise FileExistsError('Default config not found:{}'.format(ose)) + else: + self._run_pipeline = raw_args['run_pipeline'] + + # For presubmit check, do not do any image injection as for now. + # Notebook samples need to be papermilled first. + if self._is_notebook: + # Parse necessary params from config.yaml + nb_params = {} + try: + config_file = os.path.join(CONFIG_DIR, + '%s.config.yaml' % self._test_name) + with open(config_file, 'r') as f: + raw_args = yaml.safe_load(f) + test_config = yamale.make_data(config_file) + yamale.validate( + config_schema, + test_config) # If fails, a ValueError will be raised. + except yaml.YAMLError as yamlerr: + print('No legit yaml config file found, use default args:{}' + .format(yamlerr)) + except OSError as ose: + print( + 'Config file with the same name not found, use default args:{}' + .format(ose)) + else: + if 'notebook_params' in raw_args.keys(): + nb_params.update(raw_args['notebook_params']) + if 'output' in raw_args['notebook_params'].keys( + ): # output is a special param that has to be specified dynamically. + nb_params['output'] = self._sample_test_output + if 'run_pipeline' in raw_args.keys(): + self._run_pipeline = raw_args['run_pipeline'] + + pm.execute_notebook( + input_path='%s.ipynb' % self._test_name, + output_path='%s.ipynb' % self._test_name, + parameters=nb_params, + prepare_only=True) + # Convert to python script. + subprocess.call([ + 'jupyter', 'nbconvert', '--to', 'python', + '%s.ipynb' % self._test_name + ]) + + else: + subprocess.call(['python3', '%s.py' % self._test_name]) + + def _injection(self): + """Inject images for pipeline components. + + This is only valid for coimponent test + """ + pass + + def run_test(self): + self._compile() + self._injection() + + # Overriding the experiment name of pipeline runs + experiment_name = self._test_name + '-test' + os.environ['KF_PIPELINES_OVERRIDE_EXPERIMENT_NAME'] = experiment_name + + if self._is_notebook: + nbchecker = NoteBookChecker( + testname=self._test_name, + result=self._sample_test_result, + run_pipeline=self._run_pipeline, + experiment_name=experiment_name, + host=self._host, + ) + nbchecker.run() + os.chdir(TEST_DIR) + nbchecker.check() + else: + os.chdir(TEST_DIR) + input_file = os.path.join(self._work_dir, + '%s.py.yaml' % self._test_name) + + pysample_checker = PySampleChecker( + testname=self._test_name, + input=input_file, + output=self._sample_test_output, + result=self._sample_test_result, + host=self._host, + namespace=self._namespace, + experiment_name=experiment_name, + ) + pysample_checker.run() + pysample_checker.check() + + self._copy_result() class ComponentTest(SampleTest): - """ Launch a KFP sample test as component test provided its name. - - Currently follows the same logic as sample test for compatibility. - include xgboost_training_cm - """ - def __init__(self, test_name, results_gcs_dir, - gcp_image, - local_confusionmatrix_image, - local_roc_image, - target_image_prefix='', - namespace='kubeflow'): - super().__init__( - test_name=test_name, - results_gcs_dir=results_gcs_dir, - target_image_prefix=target_image_prefix, - namespace=namespace - ) - self._local_confusionmatrix_image = local_confusionmatrix_image - self._local_roc_image = local_roc_image - self._dataproc_gcp_image = gcp_image - - def _injection(self): - """Sample-specific image injection into yaml file.""" - subs = { # Tag can look like 1.0.0-rc.3, so we need both "-" and "." in the regex. - 'gcr\.io/ml-pipeline/ml-pipeline/ml-pipeline-local-confusion-matrix:(\w+|[.-])+':self._local_confusionmatrix_image, - 'gcr\.io/ml-pipeline/ml-pipeline/ml-pipeline-local-roc:(\w+|[.-])+':self._local_roc_image - } - if self._test_name == 'xgboost_training_cm': - subs.update({ - 'gcr\.io/ml-pipeline/ml-pipeline-gcp:(\w|[.-])+':self._dataproc_gcp_image - }) - - utils.file_injection('%s.py.yaml' % self._test_name, - '%s.py.yaml.tmp' % self._test_name, - subs) - else: - # Only the above sample need injection for now. - pass - utils.file_injection('%s.py.yaml' % self._test_name, - '%s.py.yaml.tmp' % self._test_name, - subs) + """Launch a KFP sample test as component test provided its name. + + Currently follows the same logic as sample test for compatibility. + include xgboost_training_cm + """ + + def __init__(self, + test_name, + results_gcs_dir, + gcp_image, + local_confusionmatrix_image, + local_roc_image, + target_image_prefix='', + namespace='kubeflow'): + super().__init__( + test_name=test_name, + results_gcs_dir=results_gcs_dir, + target_image_prefix=target_image_prefix, + namespace=namespace) + self._local_confusionmatrix_image = local_confusionmatrix_image + self._local_roc_image = local_roc_image + self._dataproc_gcp_image = gcp_image + + def _injection(self): + """Sample-specific image injection into yaml file.""" + subs = { # Tag can look like 1.0.0-rc.3, so we need both "-" and "." in the regex. + 'gcr\.io/ml-pipeline/ml-pipeline/ml-pipeline-local-confusion-matrix:(\w+|[.-])+': + self._local_confusionmatrix_image, + 'gcr\.io/ml-pipeline/ml-pipeline/ml-pipeline-local-roc:(\w+|[.-])+': + self._local_roc_image + } + if self._test_name == 'xgboost_training_cm': + subs.update({ + 'gcr\.io/ml-pipeline/ml-pipeline-gcp:(\w|[.-])+': + self._dataproc_gcp_image + }) + + utils.file_injection('%s.py.yaml' % self._test_name, + '%s.py.yaml.tmp' % self._test_name, subs) + else: + # Only the above sample need injection for now. + pass + utils.file_injection('%s.py.yaml' % self._test_name, + '%s.py.yaml.tmp' % self._test_name, subs) def main(): - """Launches either KFP sample test or component test as a command entrypoint. - - Usage: - python sample_test_launcher.py sample_test run_test arg1 arg2 to launch sample test, and - python sample_test_launcher.py component_test run_test arg1 arg2 to launch component - test. - """ - fire.Fire({ - 'sample_test': SampleTest, - 'component_test': ComponentTest - }) + """Launches either KFP sample test or component test as a command + entrypoint. + + Usage: + python sample_test_launcher.py sample_test run_test arg1 arg2 to launch sample test, and + python sample_test_launcher.py component_test run_test arg1 arg2 to launch component + test. + """ + fire.Fire({'sample_test': SampleTest, 'component_test': ComponentTest}) + if __name__ == '__main__': - main() + main() diff --git a/test/sdk-execution-tests/requirements.txt b/test/sdk-execution-tests/requirements.txt index d9df3d9858..bf44f12049 100644 --- a/test/sdk-execution-tests/requirements.txt +++ b/test/sdk-execution-tests/requirements.txt @@ -1,4 +1,3 @@ sdk/python pytest==7.1.3 pytest-asyncio-cooperative==0.28.0 -pytest-mock==3.8.2 diff --git a/test/sdk-execution-tests/sdk_execution_tests.py b/test/sdk-execution-tests/sdk_execution_tests.py index 5e3bbba19d..b05f185acb 100644 --- a/test/sdk-execution-tests/sdk_execution_tests.py +++ b/test/sdk-execution-tests/sdk_execution_tests.py @@ -65,7 +65,9 @@ def create_test_case_parameters() -> List[TestCase]: return parameters -def wait(run_result: client.client.RunPipelineResult) -> kfp_server_api.ApiRun: +def wait( + run_result: client.client.RunPipelineResult +) -> kfp_server_api.V2beta1Run: return kfp_client.wait_for_run_completion( run_id=run_result.run_id, timeout=int(TIMEOUT_SECONDS)) @@ -104,16 +106,14 @@ def get_kfp_package_path() -> str: return path -partial_component_decorator = functools.partial( +dsl.component = functools.partial( dsl.component, kfp_package_path=get_kfp_package_path()) @pytest.mark.asyncio_cooperative @pytest.mark.parametrize('test_case', create_test_case_parameters()) -async def test(test_case: TestCase, mocker) -> None: +async def test(test_case: TestCase) -> None: """Asynchronously runs all samples and test that they succeed.""" - mocker.patch.object(dsl, 'component', partial_component_decorator) - event_loop = asyncio.get_running_loop() try: run_url, run_result = run(test_case) @@ -122,4 +122,8 @@ async def test(test_case: TestCase, mocker) -> None: f'Error triggering pipeline {test_case.name}.') from e api_run = await event_loop.run_in_executor(None, wait, run_result) - assert api_run.run.status == 'Succeeded', f'Pipeline {test_case.name} ended with incorrect status: {api_run.run.status}. More info: {run_url}' + assert api_run.state == 'SUCCEEDED', f'Pipeline {test_case.name} ended with incorrect status: {api_run.state}. More info: {run_url}' + + +if __name__ == '__main__': + pytest.main() diff --git a/test/tag_for_hosted.sh b/test/tag_for_hosted.sh index ba0ce64e88..8a50cd65f2 100755 --- a/test/tag_for_hosted.sh +++ b/test/tag_for_hosted.sh @@ -100,8 +100,8 @@ docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:$MM_VER # * .cloudbuild.yaml and .release.cloudbuild.yaml # * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml # * test/tag_for_hosted.sh -docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$SEM_VER -docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.5.0 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$MM_VER +docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$SEM_VER +docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$MM_VER docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$SEM_VER docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$MM_VER diff --git a/third_party/ml-metadata/README.md b/third_party/ml-metadata/README.md index 4267604a94..1af380896f 100644 --- a/third_party/ml-metadata/README.md +++ b/third_party/ml-metadata/README.md @@ -30,6 +30,14 @@ Make sure the generated files are as expected. Update clients as described below Make sure you have installed tools and packages in [grpc golang prerequisites](https://grpc.io/docs/languages/go/quickstart/#prerequisites). +NOTE: The versions for tools are important, following is a record for when the version combination works successfully. + +```bash +apt install -y protobuf-compiler=3.15.8 +go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 +go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 +``` + #### Command ```bash diff --git a/third_party/ml-metadata/VERSION b/third_party/ml-metadata/VERSION index 3e1ad720b1..cd99d386a8 100644 --- a/third_party/ml-metadata/VERSION +++ b/third_party/ml-metadata/VERSION @@ -1 +1 @@ -1.5.0 \ No newline at end of file +1.14.0 \ No newline at end of file diff --git a/third_party/ml-metadata/go.mod b/third_party/ml-metadata/go.mod index a9e89505af..e7853a4a68 100644 --- a/third_party/ml-metadata/go.mod +++ b/third_party/ml-metadata/go.mod @@ -3,7 +3,6 @@ module github.com/kubeflow/pipelines/third_party/ml-metadata go 1.16 require ( - github.com/golang/protobuf v1.5.2 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/text v0.3.7 // indirect diff --git a/third_party/ml-metadata/go.sum b/third_party/ml-metadata/go.sum index 60f21d7c35..d4502821e9 100644 --- a/third_party/ml-metadata/go.sum +++ b/third_party/ml-metadata/go.sum @@ -68,7 +68,6 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= @@ -82,7 +81,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -91,7 +89,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -111,7 +108,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20211221231510-d629cc9a93d5 h1:v7aOwCaINsgis88/5e6DEZ6TlP7vXueAw/Ftqd5rm+w= google.golang.org/genproto v0.0.0-20211221231510-d629cc9a93d5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -122,8 +118,6 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/third_party/ml-metadata/go/ml_metadata/metadata_store.pb.go b/third_party/ml-metadata/go/ml_metadata/metadata_store.pb.go index c2988f5e0c..ed2fa881d6 100644 --- a/third_party/ml-metadata/go/ml_metadata/metadata_store.pb.go +++ b/third_party/ml-metadata/go/ml_metadata/metadata_store.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.17.3 +// protoc v3.15.8 // source: ml_metadata/proto/metadata_store.proto package ml_metadata @@ -26,6 +26,7 @@ import ( protoiface "google.golang.org/protobuf/runtime/protoiface" protoimpl "google.golang.org/protobuf/runtime/protoimpl" descriptorpb "google.golang.org/protobuf/types/descriptorpb" + anypb "google.golang.org/protobuf/types/known/anypb" structpb "google.golang.org/protobuf/types/known/structpb" reflect "reflect" sync "sync" @@ -46,7 +47,11 @@ const ( PropertyType_INT PropertyType = 1 PropertyType_DOUBLE PropertyType = 2 PropertyType_STRING PropertyType = 3 + // Prefer to use `PROTO` to store structed data since this option has + // inefficient database storage usage. PropertyType_STRUCT PropertyType = 4 + PropertyType_PROTO PropertyType = 5 + PropertyType_BOOLEAN PropertyType = 6 ) // Enum value maps for PropertyType. @@ -57,6 +62,8 @@ var ( 2: "DOUBLE", 3: "STRING", 4: "STRUCT", + 5: "PROTO", + 6: "BOOLEAN", } PropertyType_value = map[string]int32{ "UNKNOWN": 0, @@ -64,6 +71,8 @@ var ( "DOUBLE": 2, "STRING": 3, "STRUCT": 4, + "PROTO": 5, + "BOOLEAN": 6, } ) @@ -117,6 +126,17 @@ const ( Artifact_MARKED_FOR_DELETION Artifact_State = 3 // A state indicating that the artifact has been deleted. Artifact_DELETED Artifact_State = 4 + // A state indicating that the artifact has been abandoned, which may be + // due to a failed or cancelled execution. + Artifact_ABANDONED Artifact_State = 5 + // A state indicating that the artifact is a reference artifact. At + // execution start time, the orchestrator produces an output artifact for + // each output key with state PENDING. However, for an intermediate + // artifact, this first artifact's state will be REFERENCE. Intermediate + // artifacts emitted during a component's execution will copy the REFERENCE + // artifact's attributes. At the end of an execution, the artifact state + // should remain REFERENCE instead of being changed to LIVE. + Artifact_REFERENCE Artifact_State = 6 ) // Enum value maps for Artifact_State. @@ -127,6 +147,8 @@ var ( 2: "LIVE", 3: "MARKED_FOR_DELETION", 4: "DELETED", + 5: "ABANDONED", + 6: "REFERENCE", } Artifact_State_value = map[string]int32{ "UNKNOWN": 0, @@ -134,6 +156,8 @@ var ( "LIVE": 2, "MARKED_FOR_DELETION": 3, "DELETED": 4, + "ABANDONED": 5, + "REFERENCE": 6, } ) @@ -254,6 +278,7 @@ const ( Event_OUTPUT Event_Type = 4 // An output of the execution. Event_INTERNAL_INPUT Event_Type = 5 // An internal input of the execution. Event_INTERNAL_OUTPUT Event_Type = 6 // An internal output of the execution. + Event_PENDING_OUTPUT Event_Type = 7 // A pending output of the execution. ) // Enum value maps for Event_Type. @@ -266,6 +291,7 @@ var ( 4: "OUTPUT", 5: "INTERNAL_INPUT", 6: "INTERNAL_OUTPUT", + 7: "PENDING_OUTPUT", } Event_Type_value = map[string]int32{ "UNKNOWN": 0, @@ -275,6 +301,7 @@ var ( "OUTPUT": 4, "INTERNAL_INPUT": 5, "INTERNAL_OUTPUT": 6, + "PENDING_OUTPUT": 7, } ) @@ -646,7 +673,74 @@ func (x *ListOperationOptions_OrderByField_Field) UnmarshalJSON(b []byte) error // Deprecated: Use ListOperationOptions_OrderByField_Field.Descriptor instead. func (ListOperationOptions_OrderByField_Field) EnumDescriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{30, 0, 0} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{31, 0, 0} +} + +type LineageSubgraphQueryOptions_Direction int32 + +const ( + // Direction is by defult DIRECTION_UNSPECIFIED, which is equivalent to + // BIDIRECTIONAL. + LineageSubgraphQueryOptions_DIRECTION_UNSPECIFIED LineageSubgraphQueryOptions_Direction = 0 + // Indicates tracing the lineage graph by hops in upstream direction. + LineageSubgraphQueryOptions_UPSTREAM LineageSubgraphQueryOptions_Direction = 1 + // Indicates tracing the lineage graph by hops in downstream direction. + LineageSubgraphQueryOptions_DOWNSTREAM LineageSubgraphQueryOptions_Direction = 2 + // Indicates tracing the lineage graph in both directions. + LineageSubgraphQueryOptions_BIDIRECTIONAL LineageSubgraphQueryOptions_Direction = 3 +) + +// Enum value maps for LineageSubgraphQueryOptions_Direction. +var ( + LineageSubgraphQueryOptions_Direction_name = map[int32]string{ + 0: "DIRECTION_UNSPECIFIED", + 1: "UPSTREAM", + 2: "DOWNSTREAM", + 3: "BIDIRECTIONAL", + } + LineageSubgraphQueryOptions_Direction_value = map[string]int32{ + "DIRECTION_UNSPECIFIED": 0, + "UPSTREAM": 1, + "DOWNSTREAM": 2, + "BIDIRECTIONAL": 3, + } +) + +func (x LineageSubgraphQueryOptions_Direction) Enum() *LineageSubgraphQueryOptions_Direction { + p := new(LineageSubgraphQueryOptions_Direction) + *p = x + return p +} + +func (x LineageSubgraphQueryOptions_Direction) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (LineageSubgraphQueryOptions_Direction) Descriptor() protoreflect.EnumDescriptor { + return file_ml_metadata_proto_metadata_store_proto_enumTypes[9].Descriptor() +} + +func (LineageSubgraphQueryOptions_Direction) Type() protoreflect.EnumType { + return &file_ml_metadata_proto_metadata_store_proto_enumTypes[9] +} + +func (x LineageSubgraphQueryOptions_Direction) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *LineageSubgraphQueryOptions_Direction) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = LineageSubgraphQueryOptions_Direction(num) + return nil +} + +// Deprecated: Use LineageSubgraphQueryOptions_Direction.Descriptor instead. +func (LineageSubgraphQueryOptions_Direction) EnumDescriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{35, 0} } type SystemTypeExtension struct { @@ -708,6 +802,8 @@ type Value struct { // *Value_DoubleValue // *Value_StringValue // *Value_StructValue + // *Value_ProtoValue + // *Value_BoolValue Value isValue_Value `protobuf_oneof:"value"` } @@ -778,6 +874,20 @@ func (x *Value) GetStructValue() *structpb.Struct { return nil } +func (x *Value) GetProtoValue() *anypb.Any { + if x, ok := x.GetValue().(*Value_ProtoValue); ok { + return x.ProtoValue + } + return nil +} + +func (x *Value) GetBoolValue() bool { + if x, ok := x.GetValue().(*Value_BoolValue); ok { + return x.BoolValue + } + return false +} + type isValue_Value interface { isValue_Value() } @@ -798,6 +908,14 @@ type Value_StructValue struct { StructValue *structpb.Struct `protobuf:"bytes,4,opt,name=struct_value,json=structValue,oneof"` } +type Value_ProtoValue struct { + ProtoValue *anypb.Any `protobuf:"bytes,5,opt,name=proto_value,json=protoValue,oneof"` +} + +type Value_BoolValue struct { + BoolValue bool `protobuf:"varint,6,opt,name=bool_value,json=boolValue,oneof"` +} + func (*Value_IntValue) isValue_Value() {} func (*Value_DoubleValue) isValue_Value() {} @@ -806,12 +924,16 @@ func (*Value_StringValue) isValue_Value() {} func (*Value_StructValue) isValue_Value() {} +func (*Value_ProtoValue) isValue_Value() {} + +func (*Value_BoolValue) isValue_Value() {} + type Artifact struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Output only. The globally unique server generated id of the artifact. + // Output only. The unique server generated id of the artifact. Id *int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` // The client provided name of the artifact. This field is optional. If set, // it must be unique among all the artifacts of the same artifact type within @@ -825,6 +947,9 @@ type Artifact struct { // The uniform resource identifier of the physical artifact. // May be empty if there is no physical artifact. Uri *string `protobuf:"bytes,3,opt,name=uri" json:"uri,omitempty"` + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all artifacts within a database instance. + ExternalId *string `protobuf:"bytes,11,opt,name=external_id,json=externalId" json:"external_id,omitempty"` // Properties of the artifact. // Properties must be specified in the ArtifactType. Properties map[string]*Value `protobuf:"bytes,4,rep,name=properties" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` @@ -837,6 +962,8 @@ type Artifact struct { // Output only. Last update time of the artifact since epoch in millisecond // since epoch. LastUpdateTimeSinceEpoch *int64 `protobuf:"varint,10,opt,name=last_update_time_since_epoch,json=lastUpdateTimeSinceEpoch" json:"last_update_time_since_epoch,omitempty"` + // Output only. + SystemMetadata *anypb.Any `protobuf:"bytes,12,opt,name=system_metadata,json=systemMetadata" json:"system_metadata,omitempty"` } func (x *Artifact) Reset() { @@ -906,6 +1033,13 @@ func (x *Artifact) GetUri() string { return "" } +func (x *Artifact) GetExternalId() string { + if x != nil && x.ExternalId != nil { + return *x.ExternalId + } + return "" +} + func (x *Artifact) GetProperties() map[string]*Value { if x != nil { return x.Properties @@ -941,6 +1075,13 @@ func (x *Artifact) GetLastUpdateTimeSinceEpoch() int64 { return 0 } +func (x *Artifact) GetSystemMetadata() *anypb.Any { + if x != nil { + return x.SystemMetadata + } + return nil +} + type ArtifactType struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -955,6 +1096,10 @@ type ArtifactType struct { Version *string `protobuf:"bytes,4,opt,name=version" json:"version,omitempty"` // An optional description about the type. Description *string `protobuf:"bytes,5,opt,name=description" json:"description,omitempty"` + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all artifact types within a database + // instance. + ExternalId *string `protobuf:"bytes,7,opt,name=external_id,json=externalId" json:"external_id,omitempty"` // The schema of the type. // Properties are always optional in the artifact. // Properties of an artifact type can be expanded but not contracted (i.e., @@ -1026,6 +1171,13 @@ func (x *ArtifactType) GetDescription() string { return "" } +func (x *ArtifactType) GetExternalId() string { + if x != nil && x.ExternalId != nil { + return *x.ExternalId + } + return "" +} + func (x *ArtifactType) GetProperties() map[string]PropertyType { if x != nil { return x.Properties @@ -1074,40 +1226,57 @@ func (x *ArtifactType) GetBaseType() ArtifactType_SystemDefinedBaseType { // type:DECLARED_OUTPUT, // path:{step:[{"key":"my_result"}]} // } -// Other event types include INPUT/OUTPUT and INTERNAL_INPUT/_OUTPUT. +// +// Other event types include INPUT/OUTPUT, INTERNAL_INPUT/_OUTPUT and +// PENDING_OUTPUT: +// // * The INPUT/OUTPUT is an event that actually reads/writes an artifact by an // execution. The input/output artifacts may not declared in the signature, // For example, the trainer may output multiple caches of the parameters // (as an OUTPUT), then finally write the SavedModel as a DECLARED_OUTPUT. +// // * The INTERNAL_INPUT/_OUTPUT are event types which are only meaningful to // an orchestration system to keep track of the details for later debugging. // For example, a fork happened conditioning on an artifact, then an execution -// is triggered, such fork implementating may need to log the read and write +// is triggered, such fork implementing may need to log the read and write // of artifacts and may not be worth displaying to the users. // -// For instance, in the above example, +// For instance, in the above example, // -// my_result = my_execution({"data":[3,7],"schema":8}) +// my_result = my_execution({"data":[3,7],"schema":8}) // -// there is another execution (id: 15), which represents a `garbage_collection` -// step in an orchestration system +// there is another execution (id: 15), which represents a +// `garbage_collection` step in an orchestration system // -// gc_result = garbage_collection(my_result) +// gc_result = garbage_collection(my_result) // -// that cleans `my_result` if needed. The details should be invisible to the -// end users and lineage tracking. The orchestrator can emit following events: +// that cleans `my_result` if needed. The details should be invisible to the +// end users and lineage tracking. The orchestrator can emit following events: // -// { -// artifact_id: 15, -// execution_id: 15, -// type:INTERNAL_INPUT, -// } -// { -// artifact_id:16, // New artifact containing the GC job result. -// execution_id: 15, -// type:INTERNAL_OUTPUT, -// path:{step:[{"key":"gc_result"}]} -// } +// { +// artifact_id: 15, +// execution_id: 15, +// type:INTERNAL_INPUT, +// } +// { +// artifact_id:16, // New artifact containing the GC job result. +// execution_id: 15, +// type:INTERNAL_OUTPUT, +// path:{step:[{"key":"gc_result"}]} +// } +// +// * The PENDING_OUTPUT event is used to indicate that an artifact is +// tentatively associated with an active execution which has not yet been +// finalized. For example, an orchestration system can register output +// artifacts of a running execution with PENDING_OUTPUT events to indicate +// the output artifacts the execution is expected to produce. When the +// execution is finished, the final set of output artifacts can be associated +// with the exeution using OUTPUT events, and any unused artifacts which were +// previously registered with PENDING_OUTPUT events can be updated to set +// their Artifact.State to ABANDONED. +// +// Events are unique of the same +// (artifact_id, execution_id, type) combination within a metadata store. type Event struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1126,6 +1295,8 @@ type Event struct { // Time the event occurred // Epoch is Jan 1, 1970, UTC MillisecondsSinceEpoch *int64 `protobuf:"varint,5,opt,name=milliseconds_since_epoch,json=millisecondsSinceEpoch" json:"milliseconds_since_epoch,omitempty"` + // Output only. + SystemMetadata *anypb.Any `protobuf:"bytes,6,opt,name=system_metadata,json=systemMetadata" json:"system_metadata,omitempty"` } func (x *Event) Reset() { @@ -1195,12 +1366,19 @@ func (x *Event) GetMillisecondsSinceEpoch() int64 { return 0 } +func (x *Event) GetSystemMetadata() *anypb.Any { + if x != nil { + return x.SystemMetadata + } + return nil +} + type Execution struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Output only. The globally unique server generated id of the execution. + // Output only. The unique server generated id of the execution. Id *int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` // The client provided name of the execution. This field is optional. If set, // it must be unique among all the executions of the same execution type @@ -1212,6 +1390,9 @@ type Execution struct { TypeId *int64 `protobuf:"varint,2,opt,name=type_id,json=typeId" json:"type_id,omitempty"` // Output only. The name of an ExecutionType. Type *string `protobuf:"bytes,7,opt,name=type" json:"type,omitempty"` + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all executions within a database instance. + ExternalId *string `protobuf:"bytes,10,opt,name=external_id,json=externalId" json:"external_id,omitempty"` // The last known state of an execution in the system. LastKnownState *Execution_State `protobuf:"varint,3,opt,name=last_known_state,json=lastKnownState,enum=ml_metadata.Execution_State" json:"last_known_state,omitempty"` // Properties of the Execution. @@ -1223,6 +1404,8 @@ type Execution struct { CreateTimeSinceEpoch *int64 `protobuf:"varint,8,opt,name=create_time_since_epoch,json=createTimeSinceEpoch" json:"create_time_since_epoch,omitempty"` // Output only. Last update time of the execution in millisecond since epoch. LastUpdateTimeSinceEpoch *int64 `protobuf:"varint,9,opt,name=last_update_time_since_epoch,json=lastUpdateTimeSinceEpoch" json:"last_update_time_since_epoch,omitempty"` + // Output only. + SystemMetadata *anypb.Any `protobuf:"bytes,11,opt,name=system_metadata,json=systemMetadata" json:"system_metadata,omitempty"` } func (x *Execution) Reset() { @@ -1285,6 +1468,13 @@ func (x *Execution) GetType() string { return "" } +func (x *Execution) GetExternalId() string { + if x != nil && x.ExternalId != nil { + return *x.ExternalId + } + return "" +} + func (x *Execution) GetLastKnownState() Execution_State { if x != nil && x.LastKnownState != nil { return *x.LastKnownState @@ -1320,6 +1510,13 @@ func (x *Execution) GetLastUpdateTimeSinceEpoch() int64 { return 0 } +func (x *Execution) GetSystemMetadata() *anypb.Any { + if x != nil { + return x.SystemMetadata + } + return nil +} + type ExecutionType struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1334,6 +1531,10 @@ type ExecutionType struct { Version *string `protobuf:"bytes,6,opt,name=version" json:"version,omitempty"` // An optional description about the type. Description *string `protobuf:"bytes,7,opt,name=description" json:"description,omitempty"` + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all execution types within a database + // instance. + ExternalId *string `protobuf:"bytes,9,opt,name=external_id,json=externalId" json:"external_id,omitempty"` // The schema of the type. // Properties are always optional in the execution. Properties map[string]PropertyType `protobuf:"bytes,3,rep,name=properties" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value,enum=ml_metadata.PropertyType"` @@ -1424,6 +1625,13 @@ func (x *ExecutionType) GetDescription() string { return "" } +func (x *ExecutionType) GetExternalId() string { + if x != nil && x.ExternalId != nil { + return *x.ExternalId + } + return "" +} + func (x *ExecutionType) GetProperties() map[string]PropertyType { if x != nil { return x.Properties @@ -1466,6 +1674,10 @@ type ContextType struct { Version *string `protobuf:"bytes,4,opt,name=version" json:"version,omitempty"` // An optional description about the type. Description *string `protobuf:"bytes,5,opt,name=description" json:"description,omitempty"` + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all context types within a database + // instance. + ExternalId *string `protobuf:"bytes,7,opt,name=external_id,json=externalId" json:"external_id,omitempty"` // The schema of the type, e.g., name: string, owner: string // Properties are always optional in the context. // Properties of an context type can be expanded but not contracted (i.e., @@ -1538,6 +1750,13 @@ func (x *ContextType) GetDescription() string { return "" } +func (x *ContextType) GetExternalId() string { + if x != nil && x.ExternalId != nil { + return *x.ExternalId + } + return "" +} + func (x *ContextType) GetProperties() map[string]PropertyType { if x != nil { return x.Properties @@ -1557,7 +1776,7 @@ type Context struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Output Only. The globally unique server generated id of the context. + // Output Only. The unique server generated id of the context. Id *int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` // The client provided name of the context. It must be unique within a // database instance. @@ -1567,6 +1786,9 @@ type Context struct { TypeId *int64 `protobuf:"varint,2,opt,name=type_id,json=typeId" json:"type_id,omitempty"` // Output only. The name of a ContextType. Type *string `protobuf:"bytes,6,opt,name=type" json:"type,omitempty"` + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all contexts within a virtual database. + ExternalId *string `protobuf:"bytes,9,opt,name=external_id,json=externalId" json:"external_id,omitempty"` // Values of the properties, which must be specified in the ContextType. Properties map[string]*Value `protobuf:"bytes,4,rep,name=properties" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // User provided custom properties which are not defined by its type. @@ -1575,6 +1797,8 @@ type Context struct { CreateTimeSinceEpoch *int64 `protobuf:"varint,7,opt,name=create_time_since_epoch,json=createTimeSinceEpoch" json:"create_time_since_epoch,omitempty"` // Output only. Last update time of the context in millisecond since epoch. LastUpdateTimeSinceEpoch *int64 `protobuf:"varint,8,opt,name=last_update_time_since_epoch,json=lastUpdateTimeSinceEpoch" json:"last_update_time_since_epoch,omitempty"` + // Output only system metadata. + SystemMetadata *anypb.Any `protobuf:"bytes,10,opt,name=system_metadata,json=systemMetadata" json:"system_metadata,omitempty"` } func (x *Context) Reset() { @@ -1637,6 +1861,13 @@ func (x *Context) GetType() string { return "" } +func (x *Context) GetExternalId() string { + if x != nil && x.ExternalId != nil { + return *x.ExternalId + } + return "" +} + func (x *Context) GetProperties() map[string]*Value { if x != nil { return x.Properties @@ -1665,6 +1896,13 @@ func (x *Context) GetLastUpdateTimeSinceEpoch() int64 { return 0 } +func (x *Context) GetSystemMetadata() *anypb.Any { + if x != nil { + return x.SystemMetadata + } + return nil +} + // the Attribution edges between Context and Artifact instances. type Attribution struct { state protoimpl.MessageState @@ -2564,7 +2802,7 @@ type MySQLDatabaseConfig struct { Host *string `protobuf:"bytes,1,opt,name=host" json:"host,omitempty"` // The TCP Port number that the MYSQL server accepts connections on. // If unspecified, the default MYSQL port (3306) is used. - Port *uint32 `protobuf:"varint,2,opt,name=port" json:"port,omitempty"` + Port *int64 `protobuf:"varint,2,opt,name=port" json:"port,omitempty"` // The database to connect to. Must be specified. // After connecting to the MYSQL server, this database is created if not // already present unless skip_db_creation is set. @@ -2627,7 +2865,7 @@ func (x *MySQLDatabaseConfig) GetHost() string { return "" } -func (x *MySQLDatabaseConfig) GetPort() uint32 { +func (x *MySQLDatabaseConfig) GetPort() int64 { if x != nil && x.Port != nil { return *x.Port } @@ -2743,6 +2981,133 @@ func (x *SqliteMetadataSourceConfig) GetConnectionMode() SqliteMetadataSourceCon return SqliteMetadataSourceConfig_UNKNOWN } +// A config contains the parameters when using with PostgreSQLMetadatSource. +// Next index: 10 +type PostgreSQLDatabaseConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Name of host to connect to. If the host name starts with /, it is taken as + // a Unix-domain socket in the abstract namespace. + Host *string `protobuf:"bytes,1,opt,name=host" json:"host,omitempty"` + // Numeric IP address of host to connect to. If this field is provided, `host` + // field is ignored. + Hostaddr *string `protobuf:"bytes,2,opt,name=hostaddr" json:"hostaddr,omitempty"` + // Port number to connect to at the server host, or socket file name extension + // for Unix-domain connections. + Port *string `protobuf:"bytes,3,opt,name=port" json:"port,omitempty"` + // PostgreSQL user name to connect as. Defaults to be the same as the + // operating system name of the user running the application. + User *string `protobuf:"bytes,4,opt,name=user" json:"user,omitempty"` + // Password to be used if the server demands password authentication. + Password *string `protobuf:"bytes,5,opt,name=password" json:"password,omitempty"` + // Specifies the name of the file used to store passwords. + Passfile *string `protobuf:"bytes,6,opt,name=passfile" json:"passfile,omitempty"` + // The database name. Defaults to be the same as the user name. + Dbname *string `protobuf:"bytes,7,opt,name=dbname" json:"dbname,omitempty"` + // A config to skip the database creation if not exist when connecting the + // db instance. It is useful when the db creation is handled by an admin + // process, while the lib user should not issue db creation clauses. + SkipDbCreation *bool `protobuf:"varint,8,opt,name=skip_db_creation,json=skipDbCreation" json:"skip_db_creation,omitempty"` + Ssloption *PostgreSQLDatabaseConfig_SSLOptions `protobuf:"bytes,9,opt,name=ssloption" json:"ssloption,omitempty"` +} + +func (x *PostgreSQLDatabaseConfig) Reset() { + *x = PostgreSQLDatabaseConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PostgreSQLDatabaseConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PostgreSQLDatabaseConfig) ProtoMessage() {} + +func (x *PostgreSQLDatabaseConfig) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PostgreSQLDatabaseConfig.ProtoReflect.Descriptor instead. +func (*PostgreSQLDatabaseConfig) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{24} +} + +func (x *PostgreSQLDatabaseConfig) GetHost() string { + if x != nil && x.Host != nil { + return *x.Host + } + return "" +} + +func (x *PostgreSQLDatabaseConfig) GetHostaddr() string { + if x != nil && x.Hostaddr != nil { + return *x.Hostaddr + } + return "" +} + +func (x *PostgreSQLDatabaseConfig) GetPort() string { + if x != nil && x.Port != nil { + return *x.Port + } + return "" +} + +func (x *PostgreSQLDatabaseConfig) GetUser() string { + if x != nil && x.User != nil { + return *x.User + } + return "" +} + +func (x *PostgreSQLDatabaseConfig) GetPassword() string { + if x != nil && x.Password != nil { + return *x.Password + } + return "" +} + +func (x *PostgreSQLDatabaseConfig) GetPassfile() string { + if x != nil && x.Passfile != nil { + return *x.Passfile + } + return "" +} + +func (x *PostgreSQLDatabaseConfig) GetDbname() string { + if x != nil && x.Dbname != nil { + return *x.Dbname + } + return "" +} + +func (x *PostgreSQLDatabaseConfig) GetSkipDbCreation() bool { + if x != nil && x.SkipDbCreation != nil { + return *x.SkipDbCreation + } + return false +} + +func (x *PostgreSQLDatabaseConfig) GetSsloption() *PostgreSQLDatabaseConfig_SSLOptions { + if x != nil { + return x.Ssloption + } + return nil +} + type MigrationOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2778,7 +3143,7 @@ const ( func (x *MigrationOptions) Reset() { *x = MigrationOptions{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[24] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2791,7 +3156,7 @@ func (x *MigrationOptions) String() string { func (*MigrationOptions) ProtoMessage() {} func (x *MigrationOptions) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[24] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2804,7 +3169,7 @@ func (x *MigrationOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use MigrationOptions.ProtoReflect.Descriptor instead. func (*MigrationOptions) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{24} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{25} } func (x *MigrationOptions) GetEnableUpgradeMigration() bool { @@ -2833,7 +3198,7 @@ type RetryOptions struct { func (x *RetryOptions) Reset() { *x = RetryOptions{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[25] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2846,7 +3211,7 @@ func (x *RetryOptions) String() string { func (*RetryOptions) ProtoMessage() {} func (x *RetryOptions) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[25] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2859,7 +3224,7 @@ func (x *RetryOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use RetryOptions.ProtoReflect.Descriptor instead. func (*RetryOptions) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{25} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{26} } func (x *RetryOptions) GetMaxNumRetries() int64 { @@ -2880,6 +3245,7 @@ type ConnectionConfig struct { // *ConnectionConfig_FakeDatabase // *ConnectionConfig_Mysql // *ConnectionConfig_Sqlite + // *ConnectionConfig_Postgresql Config isConnectionConfig_Config `protobuf_oneof:"config"` // Options for overwriting the default retry setting when MLMD transactions // returning Aborted error. @@ -2891,7 +3257,7 @@ type ConnectionConfig struct { func (x *ConnectionConfig) Reset() { *x = ConnectionConfig{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[26] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2904,7 +3270,7 @@ func (x *ConnectionConfig) String() string { func (*ConnectionConfig) ProtoMessage() {} func (x *ConnectionConfig) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[26] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2917,7 +3283,7 @@ func (x *ConnectionConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use ConnectionConfig.ProtoReflect.Descriptor instead. func (*ConnectionConfig) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{26} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{27} } func (m *ConnectionConfig) GetConfig() isConnectionConfig_Config { @@ -2948,6 +3314,13 @@ func (x *ConnectionConfig) GetSqlite() *SqliteMetadataSourceConfig { return nil } +func (x *ConnectionConfig) GetPostgresql() *PostgreSQLDatabaseConfig { + if x, ok := x.GetConfig().(*ConnectionConfig_Postgresql); ok { + return x.Postgresql + } + return nil +} + func (x *ConnectionConfig) GetRetryOptions() *RetryOptions { if x != nil { return x.RetryOptions @@ -2971,12 +3344,19 @@ type ConnectionConfig_Sqlite struct { Sqlite *SqliteMetadataSourceConfig `protobuf:"bytes,3,opt,name=sqlite,oneof"` } +type ConnectionConfig_Postgresql struct { + // PostgreSQL database connection config. + Postgresql *PostgreSQLDatabaseConfig `protobuf:"bytes,5,opt,name=postgresql,oneof"` +} + func (*ConnectionConfig_FakeDatabase) isConnectionConfig_Config() {} func (*ConnectionConfig_Mysql) isConnectionConfig_Config() {} func (*ConnectionConfig_Sqlite) isConnectionConfig_Config() {} +func (*ConnectionConfig_Postgresql) isConnectionConfig_Config() {} + // A list of supported GRPC arguments defined in: // https://grpc.github.io/grpc/core/group__grpc__arg__keys.html type GrpcChannelArguments struct { @@ -2994,7 +3374,7 @@ type GrpcChannelArguments struct { func (x *GrpcChannelArguments) Reset() { *x = GrpcChannelArguments{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[27] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3007,7 +3387,7 @@ func (x *GrpcChannelArguments) String() string { func (*GrpcChannelArguments) ProtoMessage() {} func (x *GrpcChannelArguments) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[27] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3020,7 +3400,7 @@ func (x *GrpcChannelArguments) ProtoReflect() protoreflect.Message { // Deprecated: Use GrpcChannelArguments.ProtoReflect.Descriptor instead. func (*GrpcChannelArguments) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{27} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{28} } func (x *GrpcChannelArguments) GetMaxReceiveMessageLength() int64 { @@ -3047,7 +3427,7 @@ type MetadataStoreClientConfig struct { Host *string `protobuf:"bytes,1,opt,name=host" json:"host,omitempty"` // The TCP Port number that the gRPC server accepts connections on. // Must be specified. - Port *uint32 `protobuf:"varint,2,opt,name=port" json:"port,omitempty"` + Port *int64 `protobuf:"varint,2,opt,name=port" json:"port,omitempty"` // Configuration for a secure gRPC channel. // If not given, insecure connection is used. SslConfig *MetadataStoreClientConfig_SSLConfig `protobuf:"bytes,3,opt,name=ssl_config,json=sslConfig" json:"ssl_config,omitempty"` @@ -3063,7 +3443,7 @@ type MetadataStoreClientConfig struct { func (x *MetadataStoreClientConfig) Reset() { *x = MetadataStoreClientConfig{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[28] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3076,7 +3456,7 @@ func (x *MetadataStoreClientConfig) String() string { func (*MetadataStoreClientConfig) ProtoMessage() {} func (x *MetadataStoreClientConfig) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[28] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3089,7 +3469,7 @@ func (x *MetadataStoreClientConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use MetadataStoreClientConfig.ProtoReflect.Descriptor instead. func (*MetadataStoreClientConfig) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{28} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{29} } func (x *MetadataStoreClientConfig) GetHost() string { @@ -3099,7 +3479,7 @@ func (x *MetadataStoreClientConfig) GetHost() string { return "" } -func (x *MetadataStoreClientConfig) GetPort() uint32 { +func (x *MetadataStoreClientConfig) GetPort() int64 { if x != nil && x.Port != nil { return *x.Port } @@ -3145,7 +3525,7 @@ type MetadataStoreServerConfig struct { func (x *MetadataStoreServerConfig) Reset() { *x = MetadataStoreServerConfig{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[29] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3158,7 +3538,7 @@ func (x *MetadataStoreServerConfig) String() string { func (*MetadataStoreServerConfig) ProtoMessage() {} func (x *MetadataStoreServerConfig) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[29] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3171,7 +3551,7 @@ func (x *MetadataStoreServerConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use MetadataStoreServerConfig.ProtoReflect.Descriptor instead. func (*MetadataStoreServerConfig) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{29} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{30} } func (x *MetadataStoreServerConfig) GetConnectionConfig() *ConnectionConfig { @@ -3212,12 +3592,14 @@ type ListOperationOptions struct { // Identifies the next page of results. NextPageToken *string `protobuf:"bytes,3,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` // A boolean expression in SQL syntax that is used to specify the conditions - // on nodes' attributes and 1-hop neighborhood. + // on node attributes and directly connected assets. // // In the current implementation, filtering Artifact/Execution/Context with // the following attributes and neighborhood is supported: // - // Attributes: id:int64, type_id:int64, type:string, uri:string, name: string, + // Attributes: + // id:int64, type_id:int64, type:string, + // uri:string, name: string, external_id: string, // create_time_since_epoch:int64, last_update_time_since_epoch:int64 // state:ENUM (Artifact only) last_known_state:ENUM (Execution only) // @@ -3227,6 +3609,7 @@ type ListOperationOptions struct { // custom_properties.$name ($name is the custom property name) // attributes: the following attributes can be used // int_value: int64, double_value: double, string_value: string + // bool_value: bool // // - Context (for Artifact and Execution): // syntax: contexts_$alias ($alias can be [0-9A-Za-z_]) @@ -3254,6 +3637,7 @@ type ListOperationOptions struct { // - type = 'my_type_name' // - name = 'foo' // - type = 'bar' AND name LIKE 'foo%' + // - external_id = 'my_external_id' // - NOT(create_time_since_epoch < 1 OR last_update_time_since_epoch < 1) // // b) to filter artifacts' uri @@ -3264,10 +3648,12 @@ type ListOperationOptions struct { // c) to filter artifact's state or execution's last_known_state // - state = LIVE // - state IS NULL + // - state IN (PENDING, LIVE) // - last_known_state = RUNNING // - last_known_state != RUNNING + // - last_known_state NOT IN (FAILED, CANCELED) // - // d) to filter nodes having a specific context + // d) to filter nodes having a specific context, artifact, or execution // - contexts_a.id = 5 // - contexts_a.type = 'RunContext' // - contexts_a.name = 'my_run' @@ -3275,6 +3661,29 @@ type ListOperationOptions struct { // - contexts_a.last_update_time_since_epoch = 1626761453 // To filter nodes with conditions on multiple contexts: // - contexts_a.name = 'my_run' AND contexts_b.name = 'my_pipeline' + // To filter context with artifacts: + // - artifacts_a.id = 5 + // - artifacts_a.type = 'Dataset' + // - artifacts_a.name = 'my_dataset' + // - artifacts_a.uri = 'exact_path_string' + // - artifacts_a.state = LIVE + // - artifacts_a.state IN (PENDING, LIVE) + // - artifacts_a.external_id = "my_external_id" + // - artifacts_a.create_time_since_epoch = 1626761453 + // - artifacts_a.last_update_time_since_epoch = 1626761453 + // To filter contexts with conditions on multiple artifacts: + // - artifacts_a.name = 'my_run' AND artifacts_b.name = 'my_pipeline' + // To filter context with executions: + // - executions_a.id = 5 + // - executions_a.type = 'Dataset' + // - executions_a.name = 'my_dataset' + // - executions_a.last_known_state = RUNNING + //. - executions_a.last_known_state IN (NEW, RUNNING) + // - executions_a.external_id = "my_external_id" + // - executions_a.create_time_since_epoch = 1626761453 + // - executions_a.last_update_time_since_epoch = 1626761453 + // To filter contexts with conditions on multiple executions: + // - executions_a.name = 'my_run' AND executions_b.name = 'my_pipeline' // // e) to filter nodes condition on their properties // - properties.accuracy.double_value > 0.95 @@ -3283,7 +3692,7 @@ type ListOperationOptions struct { // other than [0-9A-Za-z_], then the name need to be backquoted, // e.g., // - properties.`my property`.int_value > 0 - // - custom_properties.`my:custom.property`.string_value = 'foo' + // - custom_properties.`my:custom.property`.bool_value = true // // f) complex query to filter both node attributes and neighborhood // - type = 'DataSet' AND @@ -3301,7 +3710,7 @@ type ListOperationOptions struct { // - events_0.milliseconds_since_epoch = 1 // to filter Executions on Events // - events_0.artifact_id = 1 - // - events_0.type = INPUT + // - events_0.type IN (INPUT, INTERNAL_INPUT) // - events_0.milliseconds_since_epoch = 1 // TODO(b/145945460) Support filtering on event step fields. FilterQuery *string `protobuf:"bytes,4,opt,name=filter_query,json=filterQuery" json:"filter_query,omitempty"` @@ -3315,7 +3724,7 @@ const ( func (x *ListOperationOptions) Reset() { *x = ListOperationOptions{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[30] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3328,7 +3737,7 @@ func (x *ListOperationOptions) String() string { func (*ListOperationOptions) ProtoMessage() {} func (x *ListOperationOptions) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[30] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3341,7 +3750,7 @@ func (x *ListOperationOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use ListOperationOptions.ProtoReflect.Descriptor instead. func (*ListOperationOptions) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{30} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{31} } func (x *ListOperationOptions) GetMaxResultSize() int32 { @@ -3403,7 +3812,7 @@ type ListOperationNextPageToken struct { func (x *ListOperationNextPageToken) Reset() { *x = ListOperationNextPageToken{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[31] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3416,7 +3825,7 @@ func (x *ListOperationNextPageToken) String() string { func (*ListOperationNextPageToken) ProtoMessage() {} func (x *ListOperationNextPageToken) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[31] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3429,7 +3838,7 @@ func (x *ListOperationNextPageToken) ProtoReflect() protoreflect.Message { // Deprecated: Use ListOperationNextPageToken.ProtoReflect.Descriptor instead. func (*ListOperationNextPageToken) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{31} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{32} } func (x *ListOperationNextPageToken) GetIdOffset() int64 { @@ -3467,12 +3876,15 @@ type TransactionOptions struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields extensionFields protoimpl.ExtensionFields + + // Transaction tag for debug use only. + Tag *string `protobuf:"bytes,1,opt,name=tag" json:"tag,omitempty"` } func (x *TransactionOptions) Reset() { *x = TransactionOptions{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[32] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3485,7 +3897,7 @@ func (x *TransactionOptions) String() string { func (*TransactionOptions) ProtoMessage() {} func (x *TransactionOptions) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[32] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3498,7 +3910,7 @@ func (x *TransactionOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use TransactionOptions.ProtoReflect.Descriptor instead. func (*TransactionOptions) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{32} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{33} } var extRange_TransactionOptions = []protoiface.ExtensionRangeV1{ @@ -3510,29 +3922,56 @@ func (*TransactionOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { return extRange_TransactionOptions } -// The query options for list lineage graph operation. It allows specifying the -// `query_nodes` of interests and the `stop_conditions` when querying a -// lineage graph. The query option is used for exporting provenance information -// from a source MLMD instance. +func (x *TransactionOptions) GetTag() string { + if x != nil && x.Tag != nil { + return *x.Tag + } + return "" +} + +// TODO(b/283852485): Deprecate GetLineageGraph API after migration to +// GetLineageSubgraph API. +// The query options for `get_lineage_graph` operation. +// `query_nodes` is a list of nodes of interest. +// Currently only artifacts are supported as `query_nodes`. +// `stop_conditions` defines the filtering rules when querying a lineage graph. +// `max_node_size` defines the total number of artifacts and executions returned +// in the subgraph. type LineageGraphQueryOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // A query to specify the nodes of interests. + // A query to specify the nodes of interest. + // `ListOperationOptions.max_result_size` sets the maximum number of nodes to + // begin with the graph search. // TODO(b/178491112) Support query_nodes for Executions. // // Types that are assignable to QueryNodes: // *LineageGraphQueryOptions_ArtifactsOptions QueryNodes isLineageGraphQueryOptions_QueryNodes `protobuf_oneof:"query_nodes"` - // A constraint option to define the boundary of the returned subgraph. + // A constraint option to define the filtering rules when querying a lineage + // graph. StopConditions *LineageGraphQueryOptions_BoundaryConstraint `protobuf:"bytes,2,opt,name=stop_conditions,json=stopConditions" json:"stop_conditions,omitempty"` + // Maximum total number of artifacts and executions in the whole returned + // lineage graph. + // If set to 0 or below, all related nodes will be returned without any + // number limitation. + // The number counts toward Artifacts and Executions. Nothing else considered. + // + // NOTE: There is no pagination supported. + MaxNodeSize *int64 `protobuf:"varint,3,opt,name=max_node_size,json=maxNodeSize,def=20" json:"max_node_size,omitempty"` } -func (x *LineageGraphQueryOptions) Reset() { - *x = LineageGraphQueryOptions{} +// Default values for LineageGraphQueryOptions fields. +const ( + Default_LineageGraphQueryOptions_MaxNodeSize = int64(20) +) + +func (x *LineageGraphQueryOptions) Reset() { + *x = LineageGraphQueryOptions{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[33] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3545,7 +3984,7 @@ func (x *LineageGraphQueryOptions) String() string { func (*LineageGraphQueryOptions) ProtoMessage() {} func (x *LineageGraphQueryOptions) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[33] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3558,7 +3997,7 @@ func (x *LineageGraphQueryOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use LineageGraphQueryOptions.ProtoReflect.Descriptor instead. func (*LineageGraphQueryOptions) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{33} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{34} } func (m *LineageGraphQueryOptions) GetQueryNodes() isLineageGraphQueryOptions_QueryNodes { @@ -3582,6 +4021,13 @@ func (x *LineageGraphQueryOptions) GetStopConditions() *LineageGraphQueryOptions return nil } +func (x *LineageGraphQueryOptions) GetMaxNodeSize() int64 { + if x != nil && x.MaxNodeSize != nil { + return *x.MaxNodeSize + } + return Default_LineageGraphQueryOptions_MaxNodeSize +} + type isLineageGraphQueryOptions_QueryNodes interface { isLineageGraphQueryOptions_QueryNodes() } @@ -3592,6 +4038,120 @@ type LineageGraphQueryOptions_ArtifactsOptions struct { func (*LineageGraphQueryOptions_ArtifactsOptions) isLineageGraphQueryOptions_QueryNodes() {} +// The query options for lineage graph tracing from a list of interested nodes. +type LineageSubgraphQueryOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to StartingNodes: + // *LineageSubgraphQueryOptions_StartingArtifacts + // *LineageSubgraphQueryOptions_StartingExecutions + StartingNodes isLineageSubgraphQueryOptions_StartingNodes `protobuf_oneof:"starting_nodes"` + // The maximum number of hops from the `starting_nodes` to traverse. + // A hop is defined as a jump to the next node following the path of + // node -> event -> next_node. + // For example, in the lineage graph a_1 -> e_1 -> a_2: + // a_2 is 2 hops away from a_1, and e_1 is 1 hop away from a_1. + // `max_num_hops` should be non-negative. + // When its value is set to 0, only the `starting_nodes` are returned. + MaxNumHops *int64 `protobuf:"varint,3,opt,name=max_num_hops,json=maxNumHops" json:"max_num_hops,omitempty"` + // The direction of lineage graph tracing, which means the direction of all + // hops in the tracing. + // An UPSTREAM hop means an expansion following the path of + // execution -> output_event -> artifact or + // artifact -> input_event -> execution + // A DOWNSTREAM hop means an expansion following the path of + // execution -> input_event -> artifact or + // artifact -> output_event -> execution + // Please refer to `Direction` for more details. + Direction *LineageSubgraphQueryOptions_Direction `protobuf:"varint,4,opt,name=direction,enum=ml_metadata.LineageSubgraphQueryOptions_Direction" json:"direction,omitempty"` +} + +func (x *LineageSubgraphQueryOptions) Reset() { + *x = LineageSubgraphQueryOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LineageSubgraphQueryOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LineageSubgraphQueryOptions) ProtoMessage() {} + +func (x *LineageSubgraphQueryOptions) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[35] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LineageSubgraphQueryOptions.ProtoReflect.Descriptor instead. +func (*LineageSubgraphQueryOptions) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{35} +} + +func (m *LineageSubgraphQueryOptions) GetStartingNodes() isLineageSubgraphQueryOptions_StartingNodes { + if m != nil { + return m.StartingNodes + } + return nil +} + +func (x *LineageSubgraphQueryOptions) GetStartingArtifacts() *LineageSubgraphQueryOptions_StartingNodes { + if x, ok := x.GetStartingNodes().(*LineageSubgraphQueryOptions_StartingArtifacts); ok { + return x.StartingArtifacts + } + return nil +} + +func (x *LineageSubgraphQueryOptions) GetStartingExecutions() *LineageSubgraphQueryOptions_StartingNodes { + if x, ok := x.GetStartingNodes().(*LineageSubgraphQueryOptions_StartingExecutions); ok { + return x.StartingExecutions + } + return nil +} + +func (x *LineageSubgraphQueryOptions) GetMaxNumHops() int64 { + if x != nil && x.MaxNumHops != nil { + return *x.MaxNumHops + } + return 0 +} + +func (x *LineageSubgraphQueryOptions) GetDirection() LineageSubgraphQueryOptions_Direction { + if x != nil && x.Direction != nil { + return *x.Direction + } + return LineageSubgraphQueryOptions_DIRECTION_UNSPECIFIED +} + +type isLineageSubgraphQueryOptions_StartingNodes interface { + isLineageSubgraphQueryOptions_StartingNodes() +} + +type LineageSubgraphQueryOptions_StartingArtifacts struct { + StartingArtifacts *LineageSubgraphQueryOptions_StartingNodes `protobuf:"bytes,1,opt,name=starting_artifacts,json=startingArtifacts,oneof"` +} + +type LineageSubgraphQueryOptions_StartingExecutions struct { + StartingExecutions *LineageSubgraphQueryOptions_StartingNodes `protobuf:"bytes,2,opt,name=starting_executions,json=startingExecutions,oneof"` +} + +func (*LineageSubgraphQueryOptions_StartingArtifacts) isLineageSubgraphQueryOptions_StartingNodes() {} + +func (*LineageSubgraphQueryOptions_StartingExecutions) isLineageSubgraphQueryOptions_StartingNodes() { +} + // A simple path (e.g. {step{key:"foo"}}) can name an artifact in the context // of an execution. type Event_Path struct { @@ -3607,7 +4167,7 @@ type Event_Path struct { func (x *Event_Path) Reset() { *x = Event_Path{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[37] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3620,7 +4180,7 @@ func (x *Event_Path) String() string { func (*Event_Path) ProtoMessage() {} func (x *Event_Path) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[37] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3657,7 +4217,7 @@ type Event_Path_Step struct { func (x *Event_Path_Step) Reset() { *x = Event_Path_Step{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[38] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3670,7 +4230,7 @@ func (x *Event_Path_Step) String() string { func (*Event_Path_Step) ProtoMessage() {} func (x *Event_Path_Step) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[38] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3747,7 +4307,7 @@ type MySQLDatabaseConfig_SSLOptions struct { func (x *MySQLDatabaseConfig_SSLOptions) Reset() { *x = MySQLDatabaseConfig_SSLOptions{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[46] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3760,7 +4320,7 @@ func (x *MySQLDatabaseConfig_SSLOptions) String() string { func (*MySQLDatabaseConfig_SSLOptions) ProtoMessage() {} func (x *MySQLDatabaseConfig_SSLOptions) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[46] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3818,6 +4378,102 @@ func (x *MySQLDatabaseConfig_SSLOptions) GetVerifyServerCert() bool { return false } +type PostgreSQLDatabaseConfig_SSLOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // disable, allow, verify-ca, verify-full, etc. Reference: + // https://www.postgresql.org/docs/current/libpq-connect.html + Sslmode *string `protobuf:"bytes,1,opt,name=sslmode" json:"sslmode,omitempty"` + // This parameter specifies the file name of the client SSL certificate, + // replacing the default ~/.postgresql/postgresql.crt. This parameter is + // ignored if an SSL connection is not made. + Sslcert *string `protobuf:"bytes,2,opt,name=sslcert" json:"sslcert,omitempty"` + // This parameter specifies the location for the secret key used for the + // client certificate. It can either specify a file name that will be used + // instead of the default ~/.postgresql/postgresql.key, this parameter is + // ignored if an SSL connection is not made. + Sslkey *string `protobuf:"bytes,3,opt,name=sslkey" json:"sslkey,omitempty"` + // This parameter specifies the password for the secret key specified in + // sslkey, allowing client certificate private keys to be stored in + // encrypted form on disk even when interactive passphrase input is not + // practical. + Sslpassword *string `protobuf:"bytes,4,opt,name=sslpassword" json:"sslpassword,omitempty"` + // This parameter specifies the name of a file containing SSL certificate + // authority (CA) certificate(s). If the file exists, the server's + // certificate will be verified to be signed by one of these authorities. + // The default is ~/.postgresql/root.crt. + Sslrootcert *string `protobuf:"bytes,5,opt,name=sslrootcert" json:"sslrootcert,omitempty"` +} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) Reset() { + *x = PostgreSQLDatabaseConfig_SSLOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PostgreSQLDatabaseConfig_SSLOptions) ProtoMessage() {} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[49] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PostgreSQLDatabaseConfig_SSLOptions.ProtoReflect.Descriptor instead. +func (*PostgreSQLDatabaseConfig_SSLOptions) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{24, 0} +} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) GetSslmode() string { + if x != nil && x.Sslmode != nil { + return *x.Sslmode + } + return "" +} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) GetSslcert() string { + if x != nil && x.Sslcert != nil { + return *x.Sslcert + } + return "" +} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) GetSslkey() string { + if x != nil && x.Sslkey != nil { + return *x.Sslkey + } + return "" +} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) GetSslpassword() string { + if x != nil && x.Sslpassword != nil { + return *x.Sslpassword + } + return "" +} + +func (x *PostgreSQLDatabaseConfig_SSLOptions) GetSslrootcert() string { + if x != nil && x.Sslrootcert != nil { + return *x.Sslrootcert + } + return "" +} + type MetadataStoreClientConfig_SSLConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3837,7 +4493,7 @@ type MetadataStoreClientConfig_SSLConfig struct { func (x *MetadataStoreClientConfig_SSLConfig) Reset() { *x = MetadataStoreClientConfig_SSLConfig{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[47] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3850,7 +4506,7 @@ func (x *MetadataStoreClientConfig_SSLConfig) String() string { func (*MetadataStoreClientConfig_SSLConfig) ProtoMessage() {} func (x *MetadataStoreClientConfig_SSLConfig) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[47] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3863,7 +4519,7 @@ func (x *MetadataStoreClientConfig_SSLConfig) ProtoReflect() protoreflect.Messag // Deprecated: Use MetadataStoreClientConfig_SSLConfig.ProtoReflect.Descriptor instead. func (*MetadataStoreClientConfig_SSLConfig) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{28, 0} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{29, 0} } func (x *MetadataStoreClientConfig_SSLConfig) GetClientKey() string { @@ -3905,7 +4561,7 @@ type MetadataStoreServerConfig_SSLConfig struct { func (x *MetadataStoreServerConfig_SSLConfig) Reset() { *x = MetadataStoreServerConfig_SSLConfig{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[48] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3918,7 +4574,7 @@ func (x *MetadataStoreServerConfig_SSLConfig) String() string { func (*MetadataStoreServerConfig_SSLConfig) ProtoMessage() {} func (x *MetadataStoreServerConfig_SSLConfig) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[48] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3931,7 +4587,7 @@ func (x *MetadataStoreServerConfig_SSLConfig) ProtoReflect() protoreflect.Messag // Deprecated: Use MetadataStoreServerConfig_SSLConfig.ProtoReflect.Descriptor instead. func (*MetadataStoreServerConfig_SSLConfig) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{29, 0} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{30, 0} } func (x *MetadataStoreServerConfig_SSLConfig) GetServerKey() string { @@ -3982,7 +4638,7 @@ const ( func (x *ListOperationOptions_OrderByField) Reset() { *x = ListOperationOptions_OrderByField{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[49] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3995,7 +4651,7 @@ func (x *ListOperationOptions_OrderByField) String() string { func (*ListOperationOptions_OrderByField) ProtoMessage() {} func (x *ListOperationOptions_OrderByField) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[49] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4008,7 +4664,7 @@ func (x *ListOperationOptions_OrderByField) ProtoReflect() protoreflect.Message // Deprecated: Use ListOperationOptions_OrderByField.ProtoReflect.Descriptor instead. func (*ListOperationOptions_OrderByField) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{30, 0} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{31, 0} } func (x *ListOperationOptions_OrderByField) GetField() ListOperationOptions_OrderByField_Field { @@ -4025,35 +4681,109 @@ func (x *ListOperationOptions_OrderByField) GetIsAsc() bool { return Default_ListOperationOptions_OrderByField_IsAsc } -// Boundary conditions to stop the traversal when return the `subgraph`. +// Filtering conditions for retrieving the lineage graph. type LineageGraphQueryOptions_BoundaryConstraint struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The maximum number of hops from the `query_nodes` to traverse. It should - // be non-negative. When zero, only the `query_nodes` are returned. + // The maximum number of hops from the `query_nodes` to traverse. + // A hop is defined as a jump to the next node following the path of + // node -> event -> next_node. + // For example, in the lineage graph a_1 -> e_1 -> a_2: + // a_2 is 2 hops away from a_1, and e_1 is 1 hop away from a_1. + // `max_num_hops` should be non-negative. + // When its value is set to 0, only the `query_nodes` are returned. MaxNumHops *int64 `protobuf:"varint,1,opt,name=max_num_hops,json=maxNumHops" json:"max_num_hops,omitempty"` - // Conditions for the boundary node in the returned the subgraph. - // Please refer to ListOperationOptions.filter_query for the syntax. - // If set, it stops traversing further at the artifacts that do not satisfy - // `boundary_artifacts` and exclude them from the returned subgraph. - // For example, to look for the models related to a DataSet and ignore - // other artifacts derived from the DataSet: - // "type = 'DataSet' OR type = 'TransformGraph' OR type = 'Model'" + // Filtering conditions for retrieving the lineage graph. + // Please refer to `ListOperationOptions.filter_query` for the syntax. + // + // If set, the `boundary_artifacts` defines which artifacts to keep in the + // returned lineage graph during the graph search. + // Artifacts that do not satisfy the `boundary_artifacts` are filtered out, + // and the subgraphs starting at them will be pruned. + // If not set, no artifacts will be filtered out. + // Taking the following lineage graph as example: + // (`a` represents an Artifact, `e` represents an Execution, each arrow + // represents a hop.) + // a_0 a_1 a_3 + // | \ / \ + // \/ \/ \/ \/ + // e_0 e_1 e_3 + // / \ + // \/ \/ + // a_2 a_4 a_5 + // \ / + // \/ \/ + // e_2 + // To query all the upstream and downstream nodes 3 hops away from a_4, + // while excluding the upstream subgraph starting at a_3, then + // `stop_conditions` can be set as: + // { + // max_num_hops: 3 + // boundary_artifacts: 'id != 3' + // } + // With the `stop_conditions`, {a_3, e_1, a_1, a_0, e_0} will be filtered + // out. + // The returned lineage graph looks like: + // e_3 + // / \ + // \/ \/ + // a_2 a_4 a_5 + // \ / + // \/ \/ + // e_2 BoundaryArtifacts *string `protobuf:"bytes,2,opt,name=boundary_artifacts,json=boundaryArtifacts" json:"boundary_artifacts,omitempty"` - // If set, it stops traversing further at the executions that do not satisfy - // `boundary_executions` and exclude them from the returned subgraph. - // For example, two trainers may be connected due to the shared datasets, - // to exclude other trainers except a particular one (e.g., id = 5): - // "type != 'Trainer' OR (type = 'Trainer' AND id = 5)" + // If set, the `boundary_executions` defines which executions to keep in the + // returned lineage graph during the graph search. + // Executions that do not satisfy the `boundary_executions` are filtered out + // and the subgraphs starting at them will be pruned. + // If not set, no executions will be filtered out. + // In the example above, to query for all the upstream and downstream nodes + // 3 hops away from a_4, while excluding the upstream subgraph and the + // downstream subgraph starting at e_3, then `stop_conditions` can be set as + // { + // max_num_hops: 3 + // boundary_executions: 'id != 3' + // } + // With the `stop_conditions`, {e_3, a_5, a_3, e_1, a_1, a_0, e_0} will be + // filtered out. + // The returned lineage graph looks like: + // a_2 a_4 + // \ / + // \/ \/ + // e_2 + // However, for the following graph: + // a_0 a_1 a_3 + // | \ / \ + // \/ \/ \/ \/ + // e_0 e_1 e_3 + // \ / \ + // \/ \/ \/ + // a_2 a_4 a_5 + // \ / + // \/ \/ + // e_2 + // With the same `stop_conditions`, only {e_3, a_5, a_0, e_0} will be + // filtered out. + // The returned lineage graph looks like: + // a_1 a_3 + // \ / + // \/ \/ + // e_1 + // \ + // \/ + // a_2 a_4 + // \ / + // \/ \/ + // e_2 BoundaryExecutions *string `protobuf:"bytes,3,opt,name=boundary_executions,json=boundaryExecutions" json:"boundary_executions,omitempty"` } func (x *LineageGraphQueryOptions_BoundaryConstraint) Reset() { *x = LineageGraphQueryOptions_BoundaryConstraint{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[50] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4066,7 +4796,7 @@ func (x *LineageGraphQueryOptions_BoundaryConstraint) String() string { func (*LineageGraphQueryOptions_BoundaryConstraint) ProtoMessage() {} func (x *LineageGraphQueryOptions_BoundaryConstraint) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[50] + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4079,7 +4809,7 @@ func (x *LineageGraphQueryOptions_BoundaryConstraint) ProtoReflect() protoreflec // Deprecated: Use LineageGraphQueryOptions_BoundaryConstraint.ProtoReflect.Descriptor instead. func (*LineageGraphQueryOptions_BoundaryConstraint) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{33, 0} + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{34, 0} } func (x *LineageGraphQueryOptions_BoundaryConstraint) GetMaxNumHops() int64 { @@ -4103,6 +4833,58 @@ func (x *LineageGraphQueryOptions_BoundaryConstraint) GetBoundaryExecutions() st return "" } +// `starting_nodes` is a list of nodes of interest to start graph tracing. +// NOTE: The maximum number of starting nodes is 100 at most. +type LineageSubgraphQueryOptions_StartingNodes struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // `filter_query` is a boolean expression in SQL syntax that is used to + // specify the conditions on starting nodes. + // Please refer to ListOperationOptions.filter_query for more details. + FilterQuery *string `protobuf:"bytes,1,opt,name=filter_query,json=filterQuery" json:"filter_query,omitempty"` +} + +func (x *LineageSubgraphQueryOptions_StartingNodes) Reset() { + *x = LineageSubgraphQueryOptions_StartingNodes{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LineageSubgraphQueryOptions_StartingNodes) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LineageSubgraphQueryOptions_StartingNodes) ProtoMessage() {} + +func (x *LineageSubgraphQueryOptions_StartingNodes) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_proto_msgTypes[54] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LineageSubgraphQueryOptions_StartingNodes.ProtoReflect.Descriptor instead. +func (*LineageSubgraphQueryOptions_StartingNodes) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_proto_rawDescGZIP(), []int{35, 0} +} + +func (x *LineageSubgraphQueryOptions_StartingNodes) GetFilterQuery() string { + if x != nil && x.FilterQuery != nil { + return *x.FilterQuery + } + return "" +} + var file_ml_metadata_proto_metadata_store_proto_extTypes = []protoimpl.ExtensionInfo{ { ExtendedType: (*descriptorpb.EnumValueOptions)(nil), @@ -4128,626 +4910,736 @@ var file_ml_metadata_proto_metadata_store_proto_rawDesc = []byte{ 0x0a, 0x26, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x32, 0x0a, 0x13, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, - 0x79, 0x70, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, - 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xb7, 0x01, 0x0a, 0x05, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, - 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, - 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3c, 0x0a, 0x0c, - 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x73, - 0x74, 0x72, 0x75, 0x63, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x22, 0xb7, 0x05, 0x0a, 0x08, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, + 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x32, 0x0a, 0x13, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x45, 0x78, + 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x91, 0x02, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, + 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, + 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, + 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x75, 0x63, + 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x37, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x5f, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, + 0x48, 0x00, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, + 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, + 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xb5, 0x06, 0x0a, 0x08, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x79, 0x70, + 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, 0x79, 0x70, 0x65, + 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x0a, 0x70, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x12, 0x58, 0x0a, 0x11, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, + 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, + 0x17, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, + 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3e, 0x0a, 0x1c, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, + 0x70, 0x6f, 0x63, 0x68, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3d, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x41, 0x6e, 0x79, 0x52, 0x0e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x1a, 0x51, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x57, 0x0a, 0x15, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x6f, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, + 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, + 0x4d, 0x41, 0x52, 0x4b, 0x45, 0x44, 0x5f, 0x46, 0x4f, 0x52, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, + 0x49, 0x4f, 0x4e, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, + 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x42, 0x41, 0x4e, 0x44, 0x4f, 0x4e, 0x45, 0x44, 0x10, + 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x46, 0x45, 0x52, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x06, + 0x22, 0xd2, 0x04, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x49, 0x64, 0x12, 0x49, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x4c, 0x0a, + 0x09, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x2f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x08, 0x62, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x1a, 0x58, 0x0a, 0x0f, 0x50, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xcd, 0x01, 0x0a, 0x15, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x26, 0x0a, 0x05, 0x55, 0x4e, 0x53, 0x45, 0x54, 0x10, 0x00, 0x1a, 0x1b, 0xaa, 0xf1, 0xfd, 0xba, + 0x0b, 0x15, 0x0a, 0x13, 0x75, 0x6e, 0x73, 0x65, 0x74, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, 0x07, 0x44, 0x41, 0x54, 0x41, 0x53, + 0x45, 0x54, 0x10, 0x01, 0x1a, 0x14, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0e, 0x0a, 0x0c, 0x6d, 0x6c, + 0x6d, 0x64, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x12, 0x1d, 0x0a, 0x05, 0x4d, 0x4f, + 0x44, 0x45, 0x4c, 0x10, 0x02, 0x1a, 0x12, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0c, 0x0a, 0x0a, 0x6d, + 0x6c, 0x6d, 0x64, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x21, 0x0a, 0x07, 0x4d, 0x45, 0x54, + 0x52, 0x49, 0x43, 0x53, 0x10, 0x03, 0x1a, 0x14, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0e, 0x0a, 0x0c, + 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x27, 0x0a, 0x0a, + 0x53, 0x54, 0x41, 0x54, 0x49, 0x53, 0x54, 0x49, 0x43, 0x53, 0x10, 0x04, 0x1a, 0x17, 0xaa, 0xf1, + 0xfd, 0xba, 0x0b, 0x11, 0x0a, 0x0f, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, + 0x73, 0x74, 0x69, 0x63, 0x73, 0x22, 0xaa, 0x04, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, + 0x1f, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, + 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, + 0x18, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, 0x73, 0x69, + 0x6e, 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x16, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x53, 0x69, 0x6e, + 0x63, 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3d, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x0e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x77, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x32, + 0x0a, 0x05, 0x73, 0x74, 0x65, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x2e, 0x53, 0x74, 0x65, 0x70, 0x52, 0x05, 0x73, 0x74, 0x65, + 0x70, 0x73, 0x1a, 0x3b, 0x0a, 0x04, 0x53, 0x74, 0x65, 0x70, 0x12, 0x16, 0x0a, 0x05, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x05, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x12, 0x12, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, + 0x90, 0x01, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x44, 0x45, 0x43, 0x4c, 0x41, 0x52, 0x45, + 0x44, 0x5f, 0x4f, 0x55, 0x54, 0x50, 0x55, 0x54, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x44, 0x45, + 0x43, 0x4c, 0x41, 0x52, 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x50, 0x55, 0x54, 0x10, 0x02, 0x12, 0x09, + 0x0a, 0x05, 0x49, 0x4e, 0x50, 0x55, 0x54, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x55, 0x54, + 0x50, 0x55, 0x54, 0x10, 0x04, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, + 0x4c, 0x5f, 0x49, 0x4e, 0x50, 0x55, 0x54, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x49, 0x4e, 0x54, + 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x5f, 0x4f, 0x55, 0x54, 0x50, 0x55, 0x54, 0x10, 0x06, 0x12, 0x12, + 0x0a, 0x0e, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x5f, 0x4f, 0x55, 0x54, 0x50, 0x55, 0x54, + 0x10, 0x07, 0x22, 0xaa, 0x06, 0x0a, 0x09, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x75, 0x72, 0x69, 0x12, 0x45, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, - 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x11, 0x63, 0x75, - 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x43, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x10, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x17, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, - 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3e, - 0x0a, 0x1c, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x1a, 0x51, - 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x1a, 0x57, 0x0a, 0x15, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x51, 0x0a, 0x05, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, - 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x08, 0x0a, - 0x04, 0x4c, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x4d, 0x41, 0x52, 0x4b, 0x45, - 0x44, 0x5f, 0x46, 0x4f, 0x52, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x03, - 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x04, 0x22, 0xb1, 0x04, - 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, - 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x49, - 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x4c, 0x0a, 0x09, 0x62, 0x61, 0x73, - 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2f, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, - 0x66, 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x62, - 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x1a, 0x58, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0xcd, 0x01, 0x0a, 0x15, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, 0x69, - 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x55, - 0x4e, 0x53, 0x45, 0x54, 0x10, 0x00, 0x1a, 0x1b, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x15, 0x0a, 0x13, - 0x75, 0x6e, 0x73, 0x65, 0x74, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, 0x07, 0x44, 0x41, 0x54, 0x41, 0x53, 0x45, 0x54, 0x10, 0x01, - 0x1a, 0x14, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0e, 0x0a, 0x0c, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x44, - 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x12, 0x1d, 0x0a, 0x05, 0x4d, 0x4f, 0x44, 0x45, 0x4c, 0x10, - 0x02, 0x1a, 0x12, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0c, 0x0a, 0x0a, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, - 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x21, 0x0a, 0x07, 0x4d, 0x45, 0x54, 0x52, 0x49, 0x43, 0x53, - 0x10, 0x03, 0x1a, 0x14, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0e, 0x0a, 0x0c, 0x6d, 0x6c, 0x6d, 0x64, - 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x27, 0x0a, 0x0a, 0x53, 0x54, 0x41, 0x54, - 0x49, 0x53, 0x54, 0x49, 0x43, 0x53, 0x10, 0x04, 0x1a, 0x17, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x11, - 0x0a, 0x0f, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, - 0x73, 0x22, 0xd6, 0x03, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x61, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, - 0x2b, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x2b, 0x0a, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x18, 0x6d, 0x69, 0x6c, - 0x6c, 0x69, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, - 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x16, 0x6d, 0x69, 0x6c, - 0x6c, 0x69, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, 0x70, - 0x6f, 0x63, 0x68, 0x1a, 0x77, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x32, 0x0a, 0x05, 0x73, - 0x74, 0x65, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x50, - 0x61, 0x74, 0x68, 0x2e, 0x53, 0x74, 0x65, 0x70, 0x52, 0x05, 0x73, 0x74, 0x65, 0x70, 0x73, 0x1a, - 0x3b, 0x0a, 0x04, 0x53, 0x74, 0x65, 0x70, 0x12, 0x16, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, - 0x12, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x7c, 0x0a, 0x04, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, - 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x44, 0x45, 0x43, 0x4c, 0x41, 0x52, 0x45, 0x44, 0x5f, 0x4f, 0x55, - 0x54, 0x50, 0x55, 0x54, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x44, 0x45, 0x43, 0x4c, 0x41, 0x52, - 0x45, 0x44, 0x5f, 0x49, 0x4e, 0x50, 0x55, 0x54, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x49, 0x4e, - 0x50, 0x55, 0x54, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x4f, 0x55, 0x54, 0x50, 0x55, 0x54, 0x10, - 0x04, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x5f, 0x49, 0x4e, - 0x50, 0x55, 0x54, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x4e, 0x41, - 0x4c, 0x5f, 0x4f, 0x55, 0x54, 0x50, 0x55, 0x54, 0x10, 0x06, 0x22, 0xca, 0x05, 0x0a, 0x09, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, - 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, - 0x79, 0x70, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x46, 0x0a, 0x10, 0x6c, 0x61, 0x73, - 0x74, 0x5f, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x46, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x59, 0x0a, 0x11, 0x63, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x10, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x17, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, - 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3e, 0x0a, 0x1c, 0x6c, - 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, - 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, - 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x1a, 0x51, 0x0a, 0x0f, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x57, - 0x0a, 0x15, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5e, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, - 0x03, 0x4e, 0x45, 0x57, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, - 0x47, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, - 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0a, 0x0a, - 0x06, 0x43, 0x41, 0x43, 0x48, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x41, 0x4e, - 0x43, 0x45, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x22, 0xd8, 0x05, 0x0a, 0x0d, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4a, 0x0a, 0x0a, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x49, 0x64, 0x12, 0x46, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, + 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x46, 0x0a, 0x0a, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x12, 0x59, 0x0a, 0x11, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x70, 0x72, 0x6f, + 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x3e, 0x0a, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x40, 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x4d, 0x0a, 0x09, 0x62, 0x61, 0x73, 0x65, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, - 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x62, 0x61, - 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x1a, 0x58, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0xef, 0x01, 0x0a, 0x15, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, 0x69, 0x6e, - 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x55, 0x4e, - 0x53, 0x45, 0x54, 0x10, 0x00, 0x1a, 0x1c, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x16, 0x0a, 0x14, 0x75, - 0x6e, 0x73, 0x65, 0x74, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x1a, 0x12, - 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0c, 0x0a, 0x0a, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x54, 0x72, 0x61, - 0x69, 0x6e, 0x12, 0x25, 0x0a, 0x09, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x10, - 0x02, 0x1a, 0x16, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x10, 0x0a, 0x0e, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x21, 0x0a, 0x07, 0x50, 0x52, 0x4f, - 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x1a, 0x14, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0e, 0x0a, 0x0c, - 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x23, 0x0a, 0x08, - 0x45, 0x56, 0x41, 0x4c, 0x55, 0x41, 0x54, 0x45, 0x10, 0x04, 0x1a, 0x15, 0xaa, 0xf1, 0xfd, 0xba, - 0x0b, 0x0f, 0x0a, 0x0d, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x75, 0x61, 0x74, - 0x65, 0x12, 0x1f, 0x0a, 0x06, 0x44, 0x45, 0x50, 0x4c, 0x4f, 0x59, 0x10, 0x05, 0x1a, 0x13, 0xaa, - 0xf1, 0xfd, 0xba, 0x0b, 0x0d, 0x0a, 0x0b, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x44, 0x65, 0x70, 0x6c, - 0x6f, 0x79, 0x22, 0x9e, 0x03, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x48, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x4b, 0x0a, 0x09, - 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x08, 0x62, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x1a, 0x58, 0x0a, 0x0f, 0x50, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x3e, 0x0a, 0x15, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, - 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x05, - 0x55, 0x4e, 0x53, 0x45, 0x54, 0x10, 0x00, 0x1a, 0x1a, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x14, 0x0a, - 0x12, 0x75, 0x6e, 0x73, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x22, 0x9c, 0x04, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x12, 0x44, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x11, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, - 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x63, - 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, - 0x35, 0x0a, 0x17, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, - 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x14, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, - 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3e, 0x0a, 0x1c, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, - 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6c, 0x61, - 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, - 0x65, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x1a, 0x51, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x57, 0x0a, 0x15, 0x43, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0x4d, 0x0a, 0x0b, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, - 0x64, 0x22, 0x4f, 0x0a, 0x0b, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x49, 0x64, 0x22, 0x47, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x49, 0x64, 0x12, 0x1b, - 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x9b, 0x04, 0x0a, 0x0c, - 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x40, 0x0a, 0x0e, - 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x43, - 0x0a, 0x0f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3c, 0x0a, - 0x0c, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x61, - 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3c, 0x0a, 0x0c, 0x61, - 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x61, 0x73, 0x73, - 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x92, 0x04, 0x0a, 0x12, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x33, 0x0a, 0x06, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x06, 0x73, - 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, - 0x00, 0x52, 0x09, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a, 0x0c, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, - 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x39, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x48, 0x00, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x12, 0x39, 0x0a, 0x04, 0x6e, 0x6f, - 0x6e, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4e, 0x6f, 0x6e, 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, - 0x04, 0x6e, 0x6f, 0x6e, 0x65, 0x12, 0x36, 0x0a, 0x03, 0x61, 0x6e, 0x79, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x41, 0x6e, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, - 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x03, 0x61, 0x6e, 0x79, 0x12, 0x3c, 0x0a, - 0x05, 0x74, 0x75, 0x70, 0x6c, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x75, 0x70, 0x6c, 0x65, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x48, 0x00, 0x52, 0x05, 0x74, 0x75, 0x70, 0x6c, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x64, - 0x69, 0x63, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x69, 0x63, 0x74, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, - 0x52, 0x04, 0x64, 0x69, 0x63, 0x74, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x5a, - 0x0a, 0x17, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, - 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3f, 0x0a, 0x0a, 0x63, 0x61, 0x6e, - 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, - 0x63, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x1e, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x41, 0x0a, 0x0b, - 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x22, - 0x53, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, - 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x07, 0x65, 0x6c, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x65, 0x6c, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x18, 0x0a, 0x16, 0x4e, 0x6f, 0x6e, 0x65, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x17, - 0x0a, 0x15, 0x41, 0x6e, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, - 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x56, 0x0a, 0x17, 0x54, 0x75, 0x70, 0x6c, 0x65, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, - 0xd7, 0x02, 0x0a, 0x16, 0x44, 0x69, 0x63, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x53, 0x0a, 0x0a, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x69, 0x63, - 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, - 0x79, 0x70, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, - 0x33, 0x0a, 0x16, 0x6e, 0x6f, 0x6e, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x6f, 0x74, - 0x5f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x13, 0x6e, 0x6f, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x69, 0x72, 0x65, 0x64, 0x12, 0x53, 0x0a, 0x15, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x63, 0x75, 0x73, + 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x35, 0x0a, + 0x17, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, + 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3e, 0x0a, 0x1c, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, + 0x70, 0x6f, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x3d, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x41, 0x6e, 0x79, 0x52, 0x0e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x1a, 0x51, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x57, 0x0a, 0x15, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x5e, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x45, 0x57, 0x10, 0x01, 0x12, 0x0b, + 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x43, + 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, + 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x41, 0x43, 0x48, 0x45, 0x44, 0x10, + 0x05, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x22, + 0xf9, 0x05, 0x0a, 0x0d, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x49, 0x64, 0x12, 0x4a, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x3e, + 0x0a, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x09, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x40, + 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x13, 0x65, 0x78, 0x74, 0x72, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x54, 0x79, 0x70, 0x65, 0x1a, 0x5e, 0x0a, 0x0f, 0x50, 0x72, 0x6f, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x4d, 0x0a, 0x09, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x2e, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x62, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x1a, + 0x58, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xef, 0x01, 0x0a, 0x15, 0x53, 0x79, + 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x55, 0x4e, 0x53, 0x45, 0x54, 0x10, 0x00, 0x1a, 0x1c, + 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x16, 0x0a, 0x14, 0x75, 0x6e, 0x73, 0x65, 0x74, 0x5f, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x05, + 0x54, 0x52, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x1a, 0x12, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0c, 0x0a, + 0x0a, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x54, 0x72, 0x61, 0x69, 0x6e, 0x12, 0x25, 0x0a, 0x09, 0x54, + 0x52, 0x41, 0x4e, 0x53, 0x46, 0x4f, 0x52, 0x4d, 0x10, 0x02, 0x1a, 0x16, 0xaa, 0xf1, 0xfd, 0xba, + 0x0b, 0x10, 0x0a, 0x0e, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, + 0x72, 0x6d, 0x12, 0x21, 0x0a, 0x07, 0x50, 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x10, 0x03, 0x1a, + 0x14, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0e, 0x0a, 0x0c, 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x50, 0x72, + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x23, 0x0a, 0x08, 0x45, 0x56, 0x41, 0x4c, 0x55, 0x41, 0x54, + 0x45, 0x10, 0x04, 0x1a, 0x15, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0f, 0x0a, 0x0d, 0x6d, 0x6c, 0x6d, + 0x64, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x75, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x06, 0x44, 0x45, + 0x50, 0x4c, 0x4f, 0x59, 0x10, 0x05, 0x1a, 0x13, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x0d, 0x0a, 0x0b, + 0x6d, 0x6c, 0x6d, 0x64, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x22, 0xbf, 0x03, 0x0a, 0x0b, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x48, 0x0a, 0x0a, + 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x28, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, + 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x4b, 0x0a, 0x09, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, + 0x64, 0x42, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x62, 0x61, 0x73, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x1a, 0x58, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3e, 0x0a, + 0x15, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x42, 0x61, + 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x05, 0x55, 0x4e, 0x53, 0x45, 0x54, 0x10, + 0x00, 0x1a, 0x1a, 0xaa, 0xf1, 0xfd, 0xba, 0x0b, 0x14, 0x0a, 0x12, 0x75, 0x6e, 0x73, 0x65, 0x74, + 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0xfc, 0x04, + 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, + 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, + 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x12, 0x44, 0x0a, 0x0a, 0x70, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x12, 0x57, 0x0a, 0x11, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x70, 0x72, 0x6f, 0x70, + 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, + 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x17, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x63, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x12, 0x3e, 0x0a, 0x1c, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x65, 0x70, 0x6f, 0x63, + 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x12, 0x3d, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, + 0x52, 0x0e, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x1a, 0x51, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x57, 0x0a, 0x15, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x35, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4d, 0x0a, 0x0b, + 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x61, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x22, 0x4f, 0x0a, 0x0b, 0x41, + 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, + 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x22, 0x47, 0x0a, 0x0d, + 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x19, 0x0a, + 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x07, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, + 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x9b, 0x04, 0x0a, 0x0c, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, + 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x40, 0x0a, 0x0e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x3d, 0x0a, + 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, + 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x73, 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, + 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3c, 0x0a, 0x0c, 0x61, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x61, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0x92, 0x04, 0x0a, 0x12, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x33, 0x0a, 0x06, 0x73, 0x69, + 0x6d, 0x70, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x06, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x12, + 0x45, 0x0a, 0x0a, 0x75, 0x6e, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x55, 0x6e, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, + 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x09, 0x75, 0x6e, 0x69, + 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x73, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, + 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x39, 0x0a, 0x04, 0x6c, 0x69, 0x73, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x04, + 0x6c, 0x69, 0x73, 0x74, 0x12, 0x39, 0x0a, 0x04, 0x6e, 0x6f, 0x6e, 0x65, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x4e, 0x6f, 0x6e, 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x04, 0x6e, 0x6f, 0x6e, 0x65, 0x12, + 0x36, 0x0a, 0x03, 0x61, 0x6e, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x79, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x48, 0x00, 0x52, 0x03, 0x61, 0x6e, 0x79, 0x12, 0x3c, 0x0a, 0x05, 0x74, 0x75, 0x70, 0x6c, 0x65, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x75, 0x70, 0x6c, 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x05, + 0x74, 0x75, 0x70, 0x6c, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x64, 0x69, 0x63, 0x74, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x44, 0x69, 0x63, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, 0x52, 0x04, 0x64, 0x69, 0x63, 0x74, + 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x5a, 0x0a, 0x17, 0x55, 0x6e, 0x69, 0x6f, + 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x3f, 0x0a, 0x0a, 0x63, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x63, 0x61, 0x6e, 0x64, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x73, 0x22, 0x63, 0x0a, 0x1e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x41, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, + 0x61, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x22, 0x53, 0x0a, 0x16, 0x4c, 0x69, 0x73, + 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x18, + 0x0a, 0x16, 0x4e, 0x6f, 0x6e, 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x17, 0x0a, 0x15, 0x41, 0x6e, 0x79, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x22, 0x56, 0x0a, 0x17, 0x54, 0x75, 0x70, 0x6c, 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x08, + 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xd7, 0x02, 0x0a, 0x16, 0x44, 0x69, + 0x63, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x53, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x44, 0x69, 0x63, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x50, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, + 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x16, 0x6e, 0x6f, 0x6e, + 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x6f, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x69, + 0x72, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x6e, 0x6f, 0x6e, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x4e, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x12, 0x53, + 0x0a, 0x15, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x14, 0x0a, 0x12, 0x46, 0x61, 0x6b, - 0x65, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, - 0xbc, 0x03, 0x0a, 0x13, 0x4d, 0x79, 0x53, 0x51, 0x4c, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, - 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, - 0x1a, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x75, - 0x73, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, - 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, - 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x63, - 0x6b, 0x65, 0x74, 0x12, 0x4c, 0x0a, 0x0b, 0x73, 0x73, 0x6c, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x79, 0x53, 0x51, 0x4c, 0x44, 0x61, 0x74, 0x61, - 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x53, 0x4c, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0a, 0x73, 0x73, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x64, 0x62, 0x5f, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x73, 0x6b, 0x69, - 0x70, 0x44, 0x62, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xa0, 0x01, 0x0a, 0x0a, - 0x53, 0x53, 0x4c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, - 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, - 0x12, 0x0e, 0x0a, 0x02, 0x63, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x63, 0x61, - 0x12, 0x16, 0x0a, 0x06, 0x63, 0x61, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x63, 0x61, 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x69, 0x70, 0x68, - 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, - 0x12, 0x2c, 0x0a, 0x12, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x76, 0x65, - 0x72, 0x69, 0x66, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x22, 0xf6, - 0x01, 0x0a, 0x1a, 0x53, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x0a, - 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x55, 0x72, 0x69, - 0x12, 0x5f, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, - 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, - 0x65, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, - 0x65, 0x22, 0x54, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, - 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, - 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x45, 0x41, 0x44, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x0d, - 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a, - 0x14, 0x52, 0x45, 0x41, 0x44, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x43, - 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x03, 0x22, 0x95, 0x01, 0x0a, 0x10, 0x4d, 0x69, 0x67, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x38, 0x0a, 0x18, - 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x75, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x6d, - 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, - 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x4d, 0x69, 0x67, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x41, 0x0a, 0x1b, 0x64, 0x6f, 0x77, 0x6e, 0x67, 0x72, - 0x61, 0x64, 0x65, 0x5f, 0x74, 0x6f, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x3a, 0x02, 0x2d, 0x31, 0x52, - 0x18, 0x64, 0x6f, 0x77, 0x6e, 0x67, 0x72, 0x61, 0x64, 0x65, 0x54, 0x6f, 0x53, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, - 0x36, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, - 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, 0xa1, 0x02, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x46, 0x0a, 0x0d, - 0x66, 0x61, 0x6b, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x46, 0x61, 0x6b, 0x65, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x0c, 0x66, 0x61, 0x6b, 0x65, 0x44, 0x61, 0x74, 0x61, - 0x62, 0x61, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x05, 0x6d, 0x79, 0x73, 0x71, 0x6c, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x4d, 0x79, 0x53, 0x51, 0x4c, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x05, 0x6d, 0x79, 0x73, 0x71, 0x6c, 0x12, 0x41, - 0x0a, 0x06, 0x73, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x71, 0x6c, + 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x13, + 0x65, 0x78, 0x74, 0x72, 0x61, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x54, + 0x79, 0x70, 0x65, 0x1a, 0x5e, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x35, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x14, 0x0a, 0x12, 0x46, 0x61, 0x6b, 0x65, 0x44, 0x61, 0x74, 0x61, 0x62, + 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xbc, 0x03, 0x0a, 0x13, 0x4d, 0x79, + 0x53, 0x51, 0x4c, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, + 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, + 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x4c, 0x0a, + 0x0b, 0x73, 0x73, 0x6c, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x4d, 0x79, 0x53, 0x51, 0x4c, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x53, 0x4c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x0a, 0x73, 0x73, 0x6c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x73, + 0x6b, 0x69, 0x70, 0x5f, 0x64, 0x62, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x44, 0x62, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xa0, 0x01, 0x0a, 0x0a, 0x53, 0x53, 0x4c, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x63, 0x61, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x63, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x61, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x70, 0x61, + 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x12, 0x76, 0x65, + 0x72, 0x69, 0x66, 0x79, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x22, 0xf6, 0x01, 0x0a, 0x1a, 0x53, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x06, 0x73, 0x71, 0x6c, 0x69, 0x74, - 0x65, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x74, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x72, 0x65, 0x74, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x88, 0x01, 0x0a, 0x14, - 0x47, 0x72, 0x70, 0x63, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3b, 0x0a, 0x1a, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x63, 0x65, - 0x69, 0x76, 0x65, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x63, - 0x65, 0x69, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x12, 0x33, 0x0a, 0x16, 0x68, 0x74, 0x74, 0x70, 0x32, 0x5f, 0x6d, 0x61, 0x78, 0x5f, 0x70, - 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6b, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x13, 0x68, 0x74, 0x74, 0x70, 0x32, 0x4d, 0x61, 0x78, 0x50, 0x69, 0x6e, 0x67, 0x53, - 0x74, 0x72, 0x69, 0x6b, 0x65, 0x73, 0x22, 0xfc, 0x02, 0x0a, 0x19, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x4f, 0x0a, 0x0a, - 0x73, 0x73, 0x6c, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x6e, + 0x61, 0x6d, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, + 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x55, 0x72, 0x69, 0x12, 0x5f, 0x0a, 0x0f, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x53, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x0e, 0x63, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x54, 0x0a, 0x0e, 0x43, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, + 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x45, + 0x41, 0x44, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x41, 0x44, + 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x41, 0x44, 0x57, + 0x52, 0x49, 0x54, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, + 0x03, 0x22, 0xdb, 0x03, 0x0a, 0x18, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4c, + 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, + 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, + 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x61, 0x64, 0x64, 0x72, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, + 0x72, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, + 0x72, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x73, 0x73, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x73, 0x73, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x16, + 0x0a, 0x06, 0x64, 0x62, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x64, 0x62, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x64, + 0x62, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0e, 0x73, 0x6b, 0x69, 0x70, 0x44, 0x62, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x4e, 0x0a, 0x09, 0x73, 0x73, 0x6c, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4c, 0x44, 0x61, 0x74, 0x61, + 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x53, 0x4c, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x09, 0x73, 0x73, 0x6c, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x1a, 0x9c, 0x01, 0x0a, 0x0a, 0x53, 0x53, 0x4c, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x73, 0x6c, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x73, 0x73, 0x6c, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x73, 0x6c, + 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x73, 0x6c, 0x63, + 0x65, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x73, 0x6c, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x73, 0x6c, 0x6b, 0x65, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x73, + 0x73, 0x6c, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x73, 0x73, 0x6c, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x20, 0x0a, + 0x0b, 0x73, 0x73, 0x6c, 0x72, 0x6f, 0x6f, 0x74, 0x63, 0x65, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x73, 0x73, 0x6c, 0x72, 0x6f, 0x6f, 0x74, 0x63, 0x65, 0x72, 0x74, 0x22, + 0x95, 0x01, 0x0a, 0x10, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x75, + 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x41, + 0x0a, 0x1b, 0x64, 0x6f, 0x77, 0x6e, 0x67, 0x72, 0x61, 0x64, 0x65, 0x5f, 0x74, 0x6f, 0x5f, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x3a, 0x02, 0x2d, 0x31, 0x52, 0x18, 0x64, 0x6f, 0x77, 0x6e, 0x67, 0x72, 0x61, + 0x64, 0x65, 0x54, 0x6f, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x36, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x79, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6e, + 0x75, 0x6d, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x22, + 0xea, 0x02, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x46, 0x0a, 0x0d, 0x66, 0x61, 0x6b, 0x65, 0x5f, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x61, 0x6b, 0x65, 0x44, 0x61, + 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x0c, + 0x66, 0x61, 0x6b, 0x65, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x05, + 0x6d, 0x79, 0x73, 0x71, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x79, 0x53, 0x51, 0x4c, 0x44, + 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, + 0x05, 0x6d, 0x79, 0x73, 0x71, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x73, 0x71, 0x6c, 0x69, 0x74, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, + 0x00, 0x52, 0x06, 0x73, 0x71, 0x6c, 0x69, 0x74, 0x65, 0x12, 0x47, 0x0a, 0x0a, 0x70, 0x6f, 0x73, + 0x74, 0x67, 0x72, 0x65, 0x73, 0x71, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x6f, 0x73, 0x74, + 0x67, 0x72, 0x65, 0x53, 0x51, 0x4c, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x0a, 0x70, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, + 0x71, 0x6c, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x74, 0x72, 0x79, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x72, 0x65, 0x74, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x88, 0x01, 0x0a, + 0x14, 0x47, 0x72, 0x70, 0x63, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3b, 0x0a, 0x1a, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x63, + 0x65, 0x69, 0x76, 0x65, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x17, 0x6d, 0x61, 0x78, 0x52, 0x65, + 0x63, 0x65, 0x69, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x12, 0x33, 0x0a, 0x16, 0x68, 0x74, 0x74, 0x70, 0x32, 0x5f, 0x6d, 0x61, 0x78, 0x5f, + 0x70, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6b, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x13, 0x68, 0x74, 0x74, 0x70, 0x32, 0x4d, 0x61, 0x78, 0x50, 0x69, 0x6e, 0x67, + 0x53, 0x74, 0x72, 0x69, 0x6b, 0x65, 0x73, 0x22, 0xfc, 0x02, 0x0a, 0x19, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x4f, 0x0a, + 0x0a, 0x73, 0x73, 0x6c, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x53, 0x4c, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4e, + 0x0a, 0x11, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x43, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x10, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2c, + 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x5f, 0x73, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x1a, 0x68, 0x0a, 0x09, + 0x53, 0x53, 0x4c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x75, 0x73, + 0x74, 0x6f, 0x6d, 0x5f, 0x63, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x75, + 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x61, 0x22, 0x94, 0x03, 0x0a, 0x19, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1d, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x4a, 0x0a, 0x11, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x10, 0x6d, 0x69, 0x67, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4f, 0x0a, 0x0a, + 0x73, 0x73, 0x6c, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x43, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x53, 0x4c, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4e, 0x0a, - 0x11, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, - 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x43, 0x68, 0x61, 0x6e, 0x6e, - 0x65, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x10, 0x63, 0x68, 0x61, - 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2c, 0x0a, - 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, - 0x73, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x1a, 0x68, 0x0a, 0x09, 0x53, - 0x53, 0x4c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, - 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x75, 0x73, 0x74, - 0x6f, 0x6d, 0x5f, 0x63, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x43, 0x61, 0x22, 0x94, 0x03, 0x0a, 0x19, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, - 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x10, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x4a, 0x0a, 0x11, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x10, 0x6d, 0x69, 0x67, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4f, 0x0a, 0x0a, 0x73, - 0x73, 0x6c, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x53, 0x4c, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x09, 0x73, 0x73, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x8d, 0x01, 0x0a, - 0x09, 0x53, 0x53, 0x4c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x75, - 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x63, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, - 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x61, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x22, 0xb0, 0x03, 0x0a, - 0x14, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2a, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x02, - 0x32, 0x30, 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, 0x69, 0x7a, - 0x65, 0x12, 0x54, 0x0a, 0x0e, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x5f, 0x62, 0x79, 0x5f, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4f, 0x72, 0x64, - 0x65, 0x72, 0x42, 0x79, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x0c, 0x6f, 0x72, 0x64, 0x65, 0x72, - 0x42, 0x79, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, - 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, - 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x1a, 0xca, 0x01, 0x0a, 0x0c, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x79, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x12, 0x4e, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x79, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x3a, 0x02, 0x49, 0x44, 0x52, 0x05, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x12, 0x1b, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x61, 0x73, 0x63, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x05, 0x69, 0x73, 0x41, 0x73, 0x63, - 0x22, 0x4d, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x15, 0x0a, 0x11, 0x46, 0x49, 0x45, - 0x4c, 0x44, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, - 0x01, 0x12, 0x14, 0x0a, 0x10, 0x4c, 0x41, 0x53, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, - 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x49, 0x44, 0x10, 0x03, 0x22, - 0xbf, 0x01, 0x0a, 0x1a, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x4e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, - 0x0a, 0x09, 0x69, 0x64, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x08, 0x69, 0x64, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x42, - 0x0a, 0x0b, 0x73, 0x65, 0x74, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0a, 0x73, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x03, 0x52, 0x09, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x64, 0x49, 0x64, - 0x73, 0x22, 0x1f, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, - 0x80, 0x02, 0x22, 0xf7, 0x02, 0x0a, 0x18, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, - 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x50, 0x0a, 0x11, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x53, 0x53, 0x4c, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x6c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x8d, 0x01, + 0x0a, 0x09, 0x53, 0x53, 0x4c, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x63, + 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x63, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x43, 0x61, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x22, 0xb0, 0x03, + 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2a, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x3a, + 0x02, 0x32, 0x30, 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, 0x69, + 0x7a, 0x65, 0x12, 0x54, 0x0a, 0x0e, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x5f, 0x62, 0x79, 0x5f, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, - 0x10, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x61, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, - 0x47, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, - 0x61, 0x69, 0x6e, 0x74, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x96, 0x01, 0x0a, 0x12, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, - 0x79, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x6d, - 0x61, 0x78, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x68, 0x6f, 0x70, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x48, 0x6f, 0x70, 0x73, 0x12, 0x2d, 0x0a, - 0x12, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x61, 0x72, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x13, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x61, 0x72, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x0d, 0x0a, - 0x0b, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x2a, 0x48, 0x0a, 0x0c, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, - 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, - 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, - 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, - 0x52, 0x55, 0x43, 0x54, 0x10, 0x04, 0x3a, 0x7b, 0x0a, 0x15, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x21, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x95, 0xde, 0xaf, 0xb7, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x54, 0x79, 0x70, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x13, - 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, - 0x69, 0x6f, 0x6e, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4f, 0x72, + 0x64, 0x65, 0x72, 0x42, 0x79, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x0c, 0x6f, 0x72, 0x64, 0x65, + 0x72, 0x42, 0x79, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, + 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x1a, 0xca, 0x01, 0x0a, 0x0c, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x79, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x12, 0x4e, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x79, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x3a, 0x02, 0x49, 0x44, 0x52, 0x05, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1b, 0x0a, 0x06, 0x69, 0x73, 0x5f, 0x61, 0x73, 0x63, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x05, 0x69, 0x73, 0x41, 0x73, + 0x63, 0x22, 0x4d, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x15, 0x0a, 0x11, 0x46, 0x49, + 0x45, 0x4c, 0x44, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, + 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x4c, 0x41, 0x53, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, + 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x02, 0x12, 0x06, 0x0a, 0x02, 0x49, 0x44, 0x10, 0x03, + 0x22, 0xbf, 0x01, 0x0a, 0x1a, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x4e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, + 0x1b, 0x0a, 0x09, 0x69, 0x64, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x08, 0x69, 0x64, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, + 0x42, 0x0a, 0x0b, 0x73, 0x65, 0x74, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0a, 0x73, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x69, 0x64, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x03, 0x52, 0x09, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x64, 0x49, + 0x64, 0x73, 0x22, 0x31, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, + 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0x9f, 0x03, 0x0a, 0x18, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, + 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x11, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x5f, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x48, 0x00, 0x52, 0x10, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x61, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x63, 0x6f, 0x6e, + 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x38, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, + 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6e, + 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6e, + 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x5f, 0x6e, + 0x6f, 0x64, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x3a, 0x02, + 0x32, 0x30, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x1a, + 0x96, 0x01, 0x0a, 0x12, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x73, + 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x6e, 0x75, + 0x6d, 0x5f, 0x68, 0x6f, 0x70, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, + 0x78, 0x4e, 0x75, 0x6d, 0x48, 0x6f, 0x70, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x62, 0x6f, 0x75, 0x6e, + 0x64, 0x61, 0x72, 0x79, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x62, 0x6f, 0x75, 0x6e, 0x64, + 0x61, 0x72, 0x79, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x0d, 0x0a, 0x0b, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, 0x84, 0x04, 0x0a, 0x1b, 0x4c, 0x69, 0x6e, 0x65, + 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x67, 0x0a, 0x12, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x69, 0x6e, 0x67, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, + 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x48, 0x00, 0x52, 0x11, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, + 0x12, 0x69, 0x0a, 0x13, 0x73, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, + 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, + 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x48, 0x00, 0x52, 0x12, 0x73, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, + 0x67, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x6d, + 0x61, 0x78, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x68, 0x6f, 0x70, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x48, 0x6f, 0x70, 0x73, 0x12, 0x50, 0x0a, + 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x32, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, + 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x1a, + 0x32, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x73, + 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x22, 0x57, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x19, 0x0a, 0x15, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x55, + 0x50, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x4f, 0x57, + 0x4e, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x42, 0x49, 0x44, + 0x49, 0x52, 0x45, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x41, 0x4c, 0x10, 0x03, 0x42, 0x10, 0x0a, 0x0e, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x2a, 0x60, + 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, + 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, + 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x02, + 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, + 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x52, 0x4f, 0x54, + 0x4f, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x4f, 0x4f, 0x4c, 0x45, 0x41, 0x4e, 0x10, 0x06, + 0x3a, 0x7b, 0x0a, 0x15, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, + 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x95, 0xde, 0xaf, + 0xb7, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x54, 0x79, 0x70, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, } var ( @@ -4762,8 +5654,8 @@ func file_ml_metadata_proto_metadata_store_proto_rawDescGZIP() []byte { return file_ml_metadata_proto_metadata_store_proto_rawDescData } -var file_ml_metadata_proto_metadata_store_proto_enumTypes = make([]protoimpl.EnumInfo, 9) -var file_ml_metadata_proto_metadata_store_proto_msgTypes = make([]protoimpl.MessageInfo, 51) +var file_ml_metadata_proto_metadata_store_proto_enumTypes = make([]protoimpl.EnumInfo, 10) +var file_ml_metadata_proto_metadata_store_proto_msgTypes = make([]protoimpl.MessageInfo, 55) var file_ml_metadata_proto_metadata_store_proto_goTypes = []interface{}{ (PropertyType)(0), // 0: ml_metadata.PropertyType (Artifact_State)(0), // 1: ml_metadata.Artifact.State @@ -4774,137 +5666,153 @@ var file_ml_metadata_proto_metadata_store_proto_goTypes = []interface{}{ (ContextType_SystemDefinedBaseType)(0), // 6: ml_metadata.ContextType.SystemDefinedBaseType (SqliteMetadataSourceConfig_ConnectionMode)(0), // 7: ml_metadata.SqliteMetadataSourceConfig.ConnectionMode (ListOperationOptions_OrderByField_Field)(0), // 8: ml_metadata.ListOperationOptions.OrderByField.Field - (*SystemTypeExtension)(nil), // 9: ml_metadata.SystemTypeExtension - (*Value)(nil), // 10: ml_metadata.Value - (*Artifact)(nil), // 11: ml_metadata.Artifact - (*ArtifactType)(nil), // 12: ml_metadata.ArtifactType - (*Event)(nil), // 13: ml_metadata.Event - (*Execution)(nil), // 14: ml_metadata.Execution - (*ExecutionType)(nil), // 15: ml_metadata.ExecutionType - (*ContextType)(nil), // 16: ml_metadata.ContextType - (*Context)(nil), // 17: ml_metadata.Context - (*Attribution)(nil), // 18: ml_metadata.Attribution - (*Association)(nil), // 19: ml_metadata.Association - (*ParentContext)(nil), // 20: ml_metadata.ParentContext - (*LineageGraph)(nil), // 21: ml_metadata.LineageGraph - (*ArtifactStructType)(nil), // 22: ml_metadata.ArtifactStructType - (*UnionArtifactStructType)(nil), // 23: ml_metadata.UnionArtifactStructType - (*IntersectionArtifactStructType)(nil), // 24: ml_metadata.IntersectionArtifactStructType - (*ListArtifactStructType)(nil), // 25: ml_metadata.ListArtifactStructType - (*NoneArtifactStructType)(nil), // 26: ml_metadata.NoneArtifactStructType - (*AnyArtifactStructType)(nil), // 27: ml_metadata.AnyArtifactStructType - (*TupleArtifactStructType)(nil), // 28: ml_metadata.TupleArtifactStructType - (*DictArtifactStructType)(nil), // 29: ml_metadata.DictArtifactStructType - (*FakeDatabaseConfig)(nil), // 30: ml_metadata.FakeDatabaseConfig - (*MySQLDatabaseConfig)(nil), // 31: ml_metadata.MySQLDatabaseConfig - (*SqliteMetadataSourceConfig)(nil), // 32: ml_metadata.SqliteMetadataSourceConfig - (*MigrationOptions)(nil), // 33: ml_metadata.MigrationOptions - (*RetryOptions)(nil), // 34: ml_metadata.RetryOptions - (*ConnectionConfig)(nil), // 35: ml_metadata.ConnectionConfig - (*GrpcChannelArguments)(nil), // 36: ml_metadata.GrpcChannelArguments - (*MetadataStoreClientConfig)(nil), // 37: ml_metadata.MetadataStoreClientConfig - (*MetadataStoreServerConfig)(nil), // 38: ml_metadata.MetadataStoreServerConfig - (*ListOperationOptions)(nil), // 39: ml_metadata.ListOperationOptions - (*ListOperationNextPageToken)(nil), // 40: ml_metadata.ListOperationNextPageToken - (*TransactionOptions)(nil), // 41: ml_metadata.TransactionOptions - (*LineageGraphQueryOptions)(nil), // 42: ml_metadata.LineageGraphQueryOptions - nil, // 43: ml_metadata.Artifact.PropertiesEntry - nil, // 44: ml_metadata.Artifact.CustomPropertiesEntry - nil, // 45: ml_metadata.ArtifactType.PropertiesEntry - (*Event_Path)(nil), // 46: ml_metadata.Event.Path - (*Event_Path_Step)(nil), // 47: ml_metadata.Event.Path.Step - nil, // 48: ml_metadata.Execution.PropertiesEntry - nil, // 49: ml_metadata.Execution.CustomPropertiesEntry - nil, // 50: ml_metadata.ExecutionType.PropertiesEntry - nil, // 51: ml_metadata.ContextType.PropertiesEntry - nil, // 52: ml_metadata.Context.PropertiesEntry - nil, // 53: ml_metadata.Context.CustomPropertiesEntry - nil, // 54: ml_metadata.DictArtifactStructType.PropertiesEntry - (*MySQLDatabaseConfig_SSLOptions)(nil), // 55: ml_metadata.MySQLDatabaseConfig.SSLOptions - (*MetadataStoreClientConfig_SSLConfig)(nil), // 56: ml_metadata.MetadataStoreClientConfig.SSLConfig - (*MetadataStoreServerConfig_SSLConfig)(nil), // 57: ml_metadata.MetadataStoreServerConfig.SSLConfig - (*ListOperationOptions_OrderByField)(nil), // 58: ml_metadata.ListOperationOptions.OrderByField - (*LineageGraphQueryOptions_BoundaryConstraint)(nil), // 59: ml_metadata.LineageGraphQueryOptions.BoundaryConstraint - (*structpb.Struct)(nil), // 60: google.protobuf.Struct - (*descriptorpb.EnumValueOptions)(nil), // 61: google.protobuf.EnumValueOptions + (LineageSubgraphQueryOptions_Direction)(0), // 9: ml_metadata.LineageSubgraphQueryOptions.Direction + (*SystemTypeExtension)(nil), // 10: ml_metadata.SystemTypeExtension + (*Value)(nil), // 11: ml_metadata.Value + (*Artifact)(nil), // 12: ml_metadata.Artifact + (*ArtifactType)(nil), // 13: ml_metadata.ArtifactType + (*Event)(nil), // 14: ml_metadata.Event + (*Execution)(nil), // 15: ml_metadata.Execution + (*ExecutionType)(nil), // 16: ml_metadata.ExecutionType + (*ContextType)(nil), // 17: ml_metadata.ContextType + (*Context)(nil), // 18: ml_metadata.Context + (*Attribution)(nil), // 19: ml_metadata.Attribution + (*Association)(nil), // 20: ml_metadata.Association + (*ParentContext)(nil), // 21: ml_metadata.ParentContext + (*LineageGraph)(nil), // 22: ml_metadata.LineageGraph + (*ArtifactStructType)(nil), // 23: ml_metadata.ArtifactStructType + (*UnionArtifactStructType)(nil), // 24: ml_metadata.UnionArtifactStructType + (*IntersectionArtifactStructType)(nil), // 25: ml_metadata.IntersectionArtifactStructType + (*ListArtifactStructType)(nil), // 26: ml_metadata.ListArtifactStructType + (*NoneArtifactStructType)(nil), // 27: ml_metadata.NoneArtifactStructType + (*AnyArtifactStructType)(nil), // 28: ml_metadata.AnyArtifactStructType + (*TupleArtifactStructType)(nil), // 29: ml_metadata.TupleArtifactStructType + (*DictArtifactStructType)(nil), // 30: ml_metadata.DictArtifactStructType + (*FakeDatabaseConfig)(nil), // 31: ml_metadata.FakeDatabaseConfig + (*MySQLDatabaseConfig)(nil), // 32: ml_metadata.MySQLDatabaseConfig + (*SqliteMetadataSourceConfig)(nil), // 33: ml_metadata.SqliteMetadataSourceConfig + (*PostgreSQLDatabaseConfig)(nil), // 34: ml_metadata.PostgreSQLDatabaseConfig + (*MigrationOptions)(nil), // 35: ml_metadata.MigrationOptions + (*RetryOptions)(nil), // 36: ml_metadata.RetryOptions + (*ConnectionConfig)(nil), // 37: ml_metadata.ConnectionConfig + (*GrpcChannelArguments)(nil), // 38: ml_metadata.GrpcChannelArguments + (*MetadataStoreClientConfig)(nil), // 39: ml_metadata.MetadataStoreClientConfig + (*MetadataStoreServerConfig)(nil), // 40: ml_metadata.MetadataStoreServerConfig + (*ListOperationOptions)(nil), // 41: ml_metadata.ListOperationOptions + (*ListOperationNextPageToken)(nil), // 42: ml_metadata.ListOperationNextPageToken + (*TransactionOptions)(nil), // 43: ml_metadata.TransactionOptions + (*LineageGraphQueryOptions)(nil), // 44: ml_metadata.LineageGraphQueryOptions + (*LineageSubgraphQueryOptions)(nil), // 45: ml_metadata.LineageSubgraphQueryOptions + nil, // 46: ml_metadata.Artifact.PropertiesEntry + nil, // 47: ml_metadata.Artifact.CustomPropertiesEntry + nil, // 48: ml_metadata.ArtifactType.PropertiesEntry + (*Event_Path)(nil), // 49: ml_metadata.Event.Path + (*Event_Path_Step)(nil), // 50: ml_metadata.Event.Path.Step + nil, // 51: ml_metadata.Execution.PropertiesEntry + nil, // 52: ml_metadata.Execution.CustomPropertiesEntry + nil, // 53: ml_metadata.ExecutionType.PropertiesEntry + nil, // 54: ml_metadata.ContextType.PropertiesEntry + nil, // 55: ml_metadata.Context.PropertiesEntry + nil, // 56: ml_metadata.Context.CustomPropertiesEntry + nil, // 57: ml_metadata.DictArtifactStructType.PropertiesEntry + (*MySQLDatabaseConfig_SSLOptions)(nil), // 58: ml_metadata.MySQLDatabaseConfig.SSLOptions + (*PostgreSQLDatabaseConfig_SSLOptions)(nil), // 59: ml_metadata.PostgreSQLDatabaseConfig.SSLOptions + (*MetadataStoreClientConfig_SSLConfig)(nil), // 60: ml_metadata.MetadataStoreClientConfig.SSLConfig + (*MetadataStoreServerConfig_SSLConfig)(nil), // 61: ml_metadata.MetadataStoreServerConfig.SSLConfig + (*ListOperationOptions_OrderByField)(nil), // 62: ml_metadata.ListOperationOptions.OrderByField + (*LineageGraphQueryOptions_BoundaryConstraint)(nil), // 63: ml_metadata.LineageGraphQueryOptions.BoundaryConstraint + (*LineageSubgraphQueryOptions_StartingNodes)(nil), // 64: ml_metadata.LineageSubgraphQueryOptions.StartingNodes + (*structpb.Struct)(nil), // 65: google.protobuf.Struct + (*anypb.Any)(nil), // 66: google.protobuf.Any + (*descriptorpb.EnumValueOptions)(nil), // 67: google.protobuf.EnumValueOptions } var file_ml_metadata_proto_metadata_store_proto_depIdxs = []int32{ - 60, // 0: ml_metadata.Value.struct_value:type_name -> google.protobuf.Struct - 43, // 1: ml_metadata.Artifact.properties:type_name -> ml_metadata.Artifact.PropertiesEntry - 44, // 2: ml_metadata.Artifact.custom_properties:type_name -> ml_metadata.Artifact.CustomPropertiesEntry - 1, // 3: ml_metadata.Artifact.state:type_name -> ml_metadata.Artifact.State - 45, // 4: ml_metadata.ArtifactType.properties:type_name -> ml_metadata.ArtifactType.PropertiesEntry - 2, // 5: ml_metadata.ArtifactType.base_type:type_name -> ml_metadata.ArtifactType.SystemDefinedBaseType - 46, // 6: ml_metadata.Event.path:type_name -> ml_metadata.Event.Path - 3, // 7: ml_metadata.Event.type:type_name -> ml_metadata.Event.Type - 4, // 8: ml_metadata.Execution.last_known_state:type_name -> ml_metadata.Execution.State - 48, // 9: ml_metadata.Execution.properties:type_name -> ml_metadata.Execution.PropertiesEntry - 49, // 10: ml_metadata.Execution.custom_properties:type_name -> ml_metadata.Execution.CustomPropertiesEntry - 50, // 11: ml_metadata.ExecutionType.properties:type_name -> ml_metadata.ExecutionType.PropertiesEntry - 22, // 12: ml_metadata.ExecutionType.input_type:type_name -> ml_metadata.ArtifactStructType - 22, // 13: ml_metadata.ExecutionType.output_type:type_name -> ml_metadata.ArtifactStructType - 5, // 14: ml_metadata.ExecutionType.base_type:type_name -> ml_metadata.ExecutionType.SystemDefinedBaseType - 51, // 15: ml_metadata.ContextType.properties:type_name -> ml_metadata.ContextType.PropertiesEntry - 6, // 16: ml_metadata.ContextType.base_type:type_name -> ml_metadata.ContextType.SystemDefinedBaseType - 52, // 17: ml_metadata.Context.properties:type_name -> ml_metadata.Context.PropertiesEntry - 53, // 18: ml_metadata.Context.custom_properties:type_name -> ml_metadata.Context.CustomPropertiesEntry - 12, // 19: ml_metadata.LineageGraph.artifact_types:type_name -> ml_metadata.ArtifactType - 15, // 20: ml_metadata.LineageGraph.execution_types:type_name -> ml_metadata.ExecutionType - 16, // 21: ml_metadata.LineageGraph.context_types:type_name -> ml_metadata.ContextType - 11, // 22: ml_metadata.LineageGraph.artifacts:type_name -> ml_metadata.Artifact - 14, // 23: ml_metadata.LineageGraph.executions:type_name -> ml_metadata.Execution - 17, // 24: ml_metadata.LineageGraph.contexts:type_name -> ml_metadata.Context - 13, // 25: ml_metadata.LineageGraph.events:type_name -> ml_metadata.Event - 18, // 26: ml_metadata.LineageGraph.attributions:type_name -> ml_metadata.Attribution - 19, // 27: ml_metadata.LineageGraph.associations:type_name -> ml_metadata.Association - 12, // 28: ml_metadata.ArtifactStructType.simple:type_name -> ml_metadata.ArtifactType - 23, // 29: ml_metadata.ArtifactStructType.union_type:type_name -> ml_metadata.UnionArtifactStructType - 24, // 30: ml_metadata.ArtifactStructType.intersection:type_name -> ml_metadata.IntersectionArtifactStructType - 25, // 31: ml_metadata.ArtifactStructType.list:type_name -> ml_metadata.ListArtifactStructType - 26, // 32: ml_metadata.ArtifactStructType.none:type_name -> ml_metadata.NoneArtifactStructType - 27, // 33: ml_metadata.ArtifactStructType.any:type_name -> ml_metadata.AnyArtifactStructType - 28, // 34: ml_metadata.ArtifactStructType.tuple:type_name -> ml_metadata.TupleArtifactStructType - 29, // 35: ml_metadata.ArtifactStructType.dict:type_name -> ml_metadata.DictArtifactStructType - 22, // 36: ml_metadata.UnionArtifactStructType.candidates:type_name -> ml_metadata.ArtifactStructType - 22, // 37: ml_metadata.IntersectionArtifactStructType.constraints:type_name -> ml_metadata.ArtifactStructType - 22, // 38: ml_metadata.ListArtifactStructType.element:type_name -> ml_metadata.ArtifactStructType - 22, // 39: ml_metadata.TupleArtifactStructType.elements:type_name -> ml_metadata.ArtifactStructType - 54, // 40: ml_metadata.DictArtifactStructType.properties:type_name -> ml_metadata.DictArtifactStructType.PropertiesEntry - 22, // 41: ml_metadata.DictArtifactStructType.extra_properties_type:type_name -> ml_metadata.ArtifactStructType - 55, // 42: ml_metadata.MySQLDatabaseConfig.ssl_options:type_name -> ml_metadata.MySQLDatabaseConfig.SSLOptions - 7, // 43: ml_metadata.SqliteMetadataSourceConfig.connection_mode:type_name -> ml_metadata.SqliteMetadataSourceConfig.ConnectionMode - 30, // 44: ml_metadata.ConnectionConfig.fake_database:type_name -> ml_metadata.FakeDatabaseConfig - 31, // 45: ml_metadata.ConnectionConfig.mysql:type_name -> ml_metadata.MySQLDatabaseConfig - 32, // 46: ml_metadata.ConnectionConfig.sqlite:type_name -> ml_metadata.SqliteMetadataSourceConfig - 34, // 47: ml_metadata.ConnectionConfig.retry_options:type_name -> ml_metadata.RetryOptions - 56, // 48: ml_metadata.MetadataStoreClientConfig.ssl_config:type_name -> ml_metadata.MetadataStoreClientConfig.SSLConfig - 36, // 49: ml_metadata.MetadataStoreClientConfig.channel_arguments:type_name -> ml_metadata.GrpcChannelArguments - 35, // 50: ml_metadata.MetadataStoreServerConfig.connection_config:type_name -> ml_metadata.ConnectionConfig - 33, // 51: ml_metadata.MetadataStoreServerConfig.migration_options:type_name -> ml_metadata.MigrationOptions - 57, // 52: ml_metadata.MetadataStoreServerConfig.ssl_config:type_name -> ml_metadata.MetadataStoreServerConfig.SSLConfig - 58, // 53: ml_metadata.ListOperationOptions.order_by_field:type_name -> ml_metadata.ListOperationOptions.OrderByField - 39, // 54: ml_metadata.ListOperationNextPageToken.set_options:type_name -> ml_metadata.ListOperationOptions - 39, // 55: ml_metadata.LineageGraphQueryOptions.artifacts_options:type_name -> ml_metadata.ListOperationOptions - 59, // 56: ml_metadata.LineageGraphQueryOptions.stop_conditions:type_name -> ml_metadata.LineageGraphQueryOptions.BoundaryConstraint - 10, // 57: ml_metadata.Artifact.PropertiesEntry.value:type_name -> ml_metadata.Value - 10, // 58: ml_metadata.Artifact.CustomPropertiesEntry.value:type_name -> ml_metadata.Value - 0, // 59: ml_metadata.ArtifactType.PropertiesEntry.value:type_name -> ml_metadata.PropertyType - 47, // 60: ml_metadata.Event.Path.steps:type_name -> ml_metadata.Event.Path.Step - 10, // 61: ml_metadata.Execution.PropertiesEntry.value:type_name -> ml_metadata.Value - 10, // 62: ml_metadata.Execution.CustomPropertiesEntry.value:type_name -> ml_metadata.Value - 0, // 63: ml_metadata.ExecutionType.PropertiesEntry.value:type_name -> ml_metadata.PropertyType - 0, // 64: ml_metadata.ContextType.PropertiesEntry.value:type_name -> ml_metadata.PropertyType - 10, // 65: ml_metadata.Context.PropertiesEntry.value:type_name -> ml_metadata.Value - 10, // 66: ml_metadata.Context.CustomPropertiesEntry.value:type_name -> ml_metadata.Value - 22, // 67: ml_metadata.DictArtifactStructType.PropertiesEntry.value:type_name -> ml_metadata.ArtifactStructType - 8, // 68: ml_metadata.ListOperationOptions.OrderByField.field:type_name -> ml_metadata.ListOperationOptions.OrderByField.Field - 61, // 69: ml_metadata.system_type_extension:extendee -> google.protobuf.EnumValueOptions - 9, // 70: ml_metadata.system_type_extension:type_name -> ml_metadata.SystemTypeExtension - 71, // [71:71] is the sub-list for method output_type - 71, // [71:71] is the sub-list for method input_type - 70, // [70:71] is the sub-list for extension type_name - 69, // [69:70] is the sub-list for extension extendee - 0, // [0:69] is the sub-list for field type_name + 65, // 0: ml_metadata.Value.struct_value:type_name -> google.protobuf.Struct + 66, // 1: ml_metadata.Value.proto_value:type_name -> google.protobuf.Any + 46, // 2: ml_metadata.Artifact.properties:type_name -> ml_metadata.Artifact.PropertiesEntry + 47, // 3: ml_metadata.Artifact.custom_properties:type_name -> ml_metadata.Artifact.CustomPropertiesEntry + 1, // 4: ml_metadata.Artifact.state:type_name -> ml_metadata.Artifact.State + 66, // 5: ml_metadata.Artifact.system_metadata:type_name -> google.protobuf.Any + 48, // 6: ml_metadata.ArtifactType.properties:type_name -> ml_metadata.ArtifactType.PropertiesEntry + 2, // 7: ml_metadata.ArtifactType.base_type:type_name -> ml_metadata.ArtifactType.SystemDefinedBaseType + 49, // 8: ml_metadata.Event.path:type_name -> ml_metadata.Event.Path + 3, // 9: ml_metadata.Event.type:type_name -> ml_metadata.Event.Type + 66, // 10: ml_metadata.Event.system_metadata:type_name -> google.protobuf.Any + 4, // 11: ml_metadata.Execution.last_known_state:type_name -> ml_metadata.Execution.State + 51, // 12: ml_metadata.Execution.properties:type_name -> ml_metadata.Execution.PropertiesEntry + 52, // 13: ml_metadata.Execution.custom_properties:type_name -> ml_metadata.Execution.CustomPropertiesEntry + 66, // 14: ml_metadata.Execution.system_metadata:type_name -> google.protobuf.Any + 53, // 15: ml_metadata.ExecutionType.properties:type_name -> ml_metadata.ExecutionType.PropertiesEntry + 23, // 16: ml_metadata.ExecutionType.input_type:type_name -> ml_metadata.ArtifactStructType + 23, // 17: ml_metadata.ExecutionType.output_type:type_name -> ml_metadata.ArtifactStructType + 5, // 18: ml_metadata.ExecutionType.base_type:type_name -> ml_metadata.ExecutionType.SystemDefinedBaseType + 54, // 19: ml_metadata.ContextType.properties:type_name -> ml_metadata.ContextType.PropertiesEntry + 6, // 20: ml_metadata.ContextType.base_type:type_name -> ml_metadata.ContextType.SystemDefinedBaseType + 55, // 21: ml_metadata.Context.properties:type_name -> ml_metadata.Context.PropertiesEntry + 56, // 22: ml_metadata.Context.custom_properties:type_name -> ml_metadata.Context.CustomPropertiesEntry + 66, // 23: ml_metadata.Context.system_metadata:type_name -> google.protobuf.Any + 13, // 24: ml_metadata.LineageGraph.artifact_types:type_name -> ml_metadata.ArtifactType + 16, // 25: ml_metadata.LineageGraph.execution_types:type_name -> ml_metadata.ExecutionType + 17, // 26: ml_metadata.LineageGraph.context_types:type_name -> ml_metadata.ContextType + 12, // 27: ml_metadata.LineageGraph.artifacts:type_name -> ml_metadata.Artifact + 15, // 28: ml_metadata.LineageGraph.executions:type_name -> ml_metadata.Execution + 18, // 29: ml_metadata.LineageGraph.contexts:type_name -> ml_metadata.Context + 14, // 30: ml_metadata.LineageGraph.events:type_name -> ml_metadata.Event + 19, // 31: ml_metadata.LineageGraph.attributions:type_name -> ml_metadata.Attribution + 20, // 32: ml_metadata.LineageGraph.associations:type_name -> ml_metadata.Association + 13, // 33: ml_metadata.ArtifactStructType.simple:type_name -> ml_metadata.ArtifactType + 24, // 34: ml_metadata.ArtifactStructType.union_type:type_name -> ml_metadata.UnionArtifactStructType + 25, // 35: ml_metadata.ArtifactStructType.intersection:type_name -> ml_metadata.IntersectionArtifactStructType + 26, // 36: ml_metadata.ArtifactStructType.list:type_name -> ml_metadata.ListArtifactStructType + 27, // 37: ml_metadata.ArtifactStructType.none:type_name -> ml_metadata.NoneArtifactStructType + 28, // 38: ml_metadata.ArtifactStructType.any:type_name -> ml_metadata.AnyArtifactStructType + 29, // 39: ml_metadata.ArtifactStructType.tuple:type_name -> ml_metadata.TupleArtifactStructType + 30, // 40: ml_metadata.ArtifactStructType.dict:type_name -> ml_metadata.DictArtifactStructType + 23, // 41: ml_metadata.UnionArtifactStructType.candidates:type_name -> ml_metadata.ArtifactStructType + 23, // 42: ml_metadata.IntersectionArtifactStructType.constraints:type_name -> ml_metadata.ArtifactStructType + 23, // 43: ml_metadata.ListArtifactStructType.element:type_name -> ml_metadata.ArtifactStructType + 23, // 44: ml_metadata.TupleArtifactStructType.elements:type_name -> ml_metadata.ArtifactStructType + 57, // 45: ml_metadata.DictArtifactStructType.properties:type_name -> ml_metadata.DictArtifactStructType.PropertiesEntry + 23, // 46: ml_metadata.DictArtifactStructType.extra_properties_type:type_name -> ml_metadata.ArtifactStructType + 58, // 47: ml_metadata.MySQLDatabaseConfig.ssl_options:type_name -> ml_metadata.MySQLDatabaseConfig.SSLOptions + 7, // 48: ml_metadata.SqliteMetadataSourceConfig.connection_mode:type_name -> ml_metadata.SqliteMetadataSourceConfig.ConnectionMode + 59, // 49: ml_metadata.PostgreSQLDatabaseConfig.ssloption:type_name -> ml_metadata.PostgreSQLDatabaseConfig.SSLOptions + 31, // 50: ml_metadata.ConnectionConfig.fake_database:type_name -> ml_metadata.FakeDatabaseConfig + 32, // 51: ml_metadata.ConnectionConfig.mysql:type_name -> ml_metadata.MySQLDatabaseConfig + 33, // 52: ml_metadata.ConnectionConfig.sqlite:type_name -> ml_metadata.SqliteMetadataSourceConfig + 34, // 53: ml_metadata.ConnectionConfig.postgresql:type_name -> ml_metadata.PostgreSQLDatabaseConfig + 36, // 54: ml_metadata.ConnectionConfig.retry_options:type_name -> ml_metadata.RetryOptions + 60, // 55: ml_metadata.MetadataStoreClientConfig.ssl_config:type_name -> ml_metadata.MetadataStoreClientConfig.SSLConfig + 38, // 56: ml_metadata.MetadataStoreClientConfig.channel_arguments:type_name -> ml_metadata.GrpcChannelArguments + 37, // 57: ml_metadata.MetadataStoreServerConfig.connection_config:type_name -> ml_metadata.ConnectionConfig + 35, // 58: ml_metadata.MetadataStoreServerConfig.migration_options:type_name -> ml_metadata.MigrationOptions + 61, // 59: ml_metadata.MetadataStoreServerConfig.ssl_config:type_name -> ml_metadata.MetadataStoreServerConfig.SSLConfig + 62, // 60: ml_metadata.ListOperationOptions.order_by_field:type_name -> ml_metadata.ListOperationOptions.OrderByField + 41, // 61: ml_metadata.ListOperationNextPageToken.set_options:type_name -> ml_metadata.ListOperationOptions + 41, // 62: ml_metadata.LineageGraphQueryOptions.artifacts_options:type_name -> ml_metadata.ListOperationOptions + 63, // 63: ml_metadata.LineageGraphQueryOptions.stop_conditions:type_name -> ml_metadata.LineageGraphQueryOptions.BoundaryConstraint + 64, // 64: ml_metadata.LineageSubgraphQueryOptions.starting_artifacts:type_name -> ml_metadata.LineageSubgraphQueryOptions.StartingNodes + 64, // 65: ml_metadata.LineageSubgraphQueryOptions.starting_executions:type_name -> ml_metadata.LineageSubgraphQueryOptions.StartingNodes + 9, // 66: ml_metadata.LineageSubgraphQueryOptions.direction:type_name -> ml_metadata.LineageSubgraphQueryOptions.Direction + 11, // 67: ml_metadata.Artifact.PropertiesEntry.value:type_name -> ml_metadata.Value + 11, // 68: ml_metadata.Artifact.CustomPropertiesEntry.value:type_name -> ml_metadata.Value + 0, // 69: ml_metadata.ArtifactType.PropertiesEntry.value:type_name -> ml_metadata.PropertyType + 50, // 70: ml_metadata.Event.Path.steps:type_name -> ml_metadata.Event.Path.Step + 11, // 71: ml_metadata.Execution.PropertiesEntry.value:type_name -> ml_metadata.Value + 11, // 72: ml_metadata.Execution.CustomPropertiesEntry.value:type_name -> ml_metadata.Value + 0, // 73: ml_metadata.ExecutionType.PropertiesEntry.value:type_name -> ml_metadata.PropertyType + 0, // 74: ml_metadata.ContextType.PropertiesEntry.value:type_name -> ml_metadata.PropertyType + 11, // 75: ml_metadata.Context.PropertiesEntry.value:type_name -> ml_metadata.Value + 11, // 76: ml_metadata.Context.CustomPropertiesEntry.value:type_name -> ml_metadata.Value + 23, // 77: ml_metadata.DictArtifactStructType.PropertiesEntry.value:type_name -> ml_metadata.ArtifactStructType + 8, // 78: ml_metadata.ListOperationOptions.OrderByField.field:type_name -> ml_metadata.ListOperationOptions.OrderByField.Field + 67, // 79: ml_metadata.system_type_extension:extendee -> google.protobuf.EnumValueOptions + 10, // 80: ml_metadata.system_type_extension:type_name -> ml_metadata.SystemTypeExtension + 81, // [81:81] is the sub-list for method output_type + 81, // [81:81] is the sub-list for method input_type + 80, // [80:81] is the sub-list for extension type_name + 79, // [79:80] is the sub-list for extension extendee + 0, // [0:79] is the sub-list for field type_name } func init() { file_ml_metadata_proto_metadata_store_proto_init() } @@ -5202,7 +6110,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MigrationOptions); i { + switch v := v.(*PostgreSQLDatabaseConfig); i { case 0: return &v.state case 1: @@ -5214,7 +6122,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RetryOptions); i { + switch v := v.(*MigrationOptions); i { case 0: return &v.state case 1: @@ -5226,7 +6134,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ConnectionConfig); i { + switch v := v.(*RetryOptions); i { case 0: return &v.state case 1: @@ -5238,7 +6146,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GrpcChannelArguments); i { + switch v := v.(*ConnectionConfig); i { case 0: return &v.state case 1: @@ -5250,7 +6158,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MetadataStoreClientConfig); i { + switch v := v.(*GrpcChannelArguments); i { case 0: return &v.state case 1: @@ -5262,7 +6170,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MetadataStoreServerConfig); i { + switch v := v.(*MetadataStoreClientConfig); i { case 0: return &v.state case 1: @@ -5274,7 +6182,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListOperationOptions); i { + switch v := v.(*MetadataStoreServerConfig); i { case 0: return &v.state case 1: @@ -5286,7 +6194,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListOperationNextPageToken); i { + switch v := v.(*ListOperationOptions); i { case 0: return &v.state case 1: @@ -5298,6 +6206,18 @@ func file_ml_metadata_proto_metadata_store_proto_init() { } } file_ml_metadata_proto_metadata_store_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListOperationNextPageToken); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TransactionOptions); i { case 0: return &v.state @@ -5311,7 +6231,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*LineageGraphQueryOptions); i { case 0: return &v.state @@ -5323,7 +6243,19 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LineageSubgraphQueryOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Event_Path); i { case 0: return &v.state @@ -5335,7 +6267,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Event_Path_Step); i { case 0: return &v.state @@ -5347,7 +6279,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MySQLDatabaseConfig_SSLOptions); i { case 0: return &v.state @@ -5359,7 +6291,19 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PostgreSQLDatabaseConfig_SSLOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetadataStoreClientConfig_SSLConfig); i { case 0: return &v.state @@ -5371,7 +6315,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*MetadataStoreServerConfig_SSLConfig); i { case 0: return &v.state @@ -5383,7 +6327,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListOperationOptions_OrderByField); i { case 0: return &v.state @@ -5395,7 +6339,7 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { + file_ml_metadata_proto_metadata_store_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*LineageGraphQueryOptions_BoundaryConstraint); i { case 0: return &v.state @@ -5407,12 +6351,26 @@ func file_ml_metadata_proto_metadata_store_proto_init() { return nil } } + file_ml_metadata_proto_metadata_store_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LineageSubgraphQueryOptions_StartingNodes); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_ml_metadata_proto_metadata_store_proto_msgTypes[1].OneofWrappers = []interface{}{ (*Value_IntValue)(nil), (*Value_DoubleValue)(nil), (*Value_StringValue)(nil), (*Value_StructValue)(nil), + (*Value_ProtoValue)(nil), + (*Value_BoolValue)(nil), } file_ml_metadata_proto_metadata_store_proto_msgTypes[13].OneofWrappers = []interface{}{ (*ArtifactStructType_Simple)(nil), @@ -5424,15 +6382,20 @@ func file_ml_metadata_proto_metadata_store_proto_init() { (*ArtifactStructType_Tuple)(nil), (*ArtifactStructType_Dict)(nil), } - file_ml_metadata_proto_metadata_store_proto_msgTypes[26].OneofWrappers = []interface{}{ + file_ml_metadata_proto_metadata_store_proto_msgTypes[27].OneofWrappers = []interface{}{ (*ConnectionConfig_FakeDatabase)(nil), (*ConnectionConfig_Mysql)(nil), (*ConnectionConfig_Sqlite)(nil), + (*ConnectionConfig_Postgresql)(nil), } - file_ml_metadata_proto_metadata_store_proto_msgTypes[33].OneofWrappers = []interface{}{ + file_ml_metadata_proto_metadata_store_proto_msgTypes[34].OneofWrappers = []interface{}{ (*LineageGraphQueryOptions_ArtifactsOptions)(nil), } - file_ml_metadata_proto_metadata_store_proto_msgTypes[38].OneofWrappers = []interface{}{ + file_ml_metadata_proto_metadata_store_proto_msgTypes[35].OneofWrappers = []interface{}{ + (*LineageSubgraphQueryOptions_StartingArtifacts)(nil), + (*LineageSubgraphQueryOptions_StartingExecutions)(nil), + } + file_ml_metadata_proto_metadata_store_proto_msgTypes[40].OneofWrappers = []interface{}{ (*Event_Path_Step_Index)(nil), (*Event_Path_Step_Key)(nil), } @@ -5441,8 +6404,8 @@ func file_ml_metadata_proto_metadata_store_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_ml_metadata_proto_metadata_store_proto_rawDesc, - NumEnums: 9, - NumMessages: 51, + NumEnums: 10, + NumMessages: 55, NumExtensions: 1, NumServices: 0, }, diff --git a/third_party/ml-metadata/go/ml_metadata/metadata_store_service.pb.go b/third_party/ml-metadata/go/ml_metadata/metadata_store_service.pb.go index 35c5beef3f..3bf18dd322 100644 --- a/third_party/ml-metadata/go/ml_metadata/metadata_store_service.pb.go +++ b/third_party/ml-metadata/go/ml_metadata/metadata_store_service.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.17.3 +// protoc v3.15.8 // source: ml_metadata/proto/metadata_store_service.proto package ml_metadata @@ -24,6 +24,7 @@ package ml_metadata import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + fieldmaskpb "google.golang.org/protobuf/types/known/fieldmaskpb" reflect "reflect" sync "sync" ) @@ -296,6 +297,92 @@ type PutArtifactsRequest struct { Artifacts []*Artifact `protobuf:"bytes,1,rep,name=artifacts" json:"artifacts,omitempty"` // Additional options to change the behavior of the method. Options *PutArtifactsRequest_Options `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + // FieldMask for artifacts in the PUT update + // If `artifact.id` is not specified, it means a new artifact will be created + // and `update_mask` will not be applied to the creation. + // If `update_mask` is empty, update the artifacts as a whole. + // If `update_mask` is not empty, only update fields or properties specified + // in `update_mask`. + // Example request protos: + // 1. Examples that update `properties` / `custom_properties`: + // 1.1 Add a <'key', 'val'> pair into `custom_properties`: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // string_value: "val" + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 1.2 Set `custom_properties['key'].bool_value` to true: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // bool_value: true + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 1.3 Delete the complete <'key', 'val'> pair from `custom_properties`: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // custom_properties {} + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 2. Examples that update fields such as `uri`, `external_id`, etc: + // 2.1 Update `external_id` field: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // external_id: "new_value" + // } + // update_mask { + // paths: "external_id" + // } + // } + // 2.2 Set `uri` field: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // uri: "set_value" + // } + // update_mask { + // paths: "uri" + // } + // } + // If `paths: "properties"` or `paths: "custom_properties"` are added to + // `update_mask`, the key-level updates will be ignored and we only perform + // field-level updates on the all `properties`/`custom_properties`. + // For example: + // If the mask is: {"properties", "properties.key1"}, the field path + // "properties.key1" will be ignored and all `properties` will be updated. + // (Do not suggest) + // If the mask is {"properties", "external_id"}, all + // `properties` and field `external_id` will be updated. (Do not suggest) + UpdateMask *fieldmaskpb.FieldMask `protobuf:"bytes,4,opt,name=update_mask,json=updateMask" json:"update_mask,omitempty"` } func (x *PutArtifactsRequest) Reset() { @@ -344,6 +431,20 @@ func (x *PutArtifactsRequest) GetOptions() *PutArtifactsRequest_Options { return nil } +func (x *PutArtifactsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +func (x *PutArtifactsRequest) GetUpdateMask() *fieldmaskpb.FieldMask { + if x != nil { + return x.UpdateMask + } + return nil +} + type PutArtifactsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -417,6 +518,8 @@ type PutArtifactTypeRequest struct { CanDeleteFields *bool `protobuf:"varint,3,opt,name=can_delete_fields,json=canDeleteFields" json:"can_delete_fields,omitempty"` // Deprecated: Do not use. AllFieldsMatch *bool `protobuf:"varint,4,opt,name=all_fields_match,json=allFieldsMatch,def=1" json:"all_fields_match,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,6,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } // Default values for PutArtifactTypeRequest fields. @@ -493,6 +596,13 @@ func (x *PutArtifactTypeRequest) GetAllFieldsMatch() bool { return Default_PutArtifactTypeRequest_AllFieldsMatch } +func (x *PutArtifactTypeRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutArtifactTypeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -547,6 +657,44 @@ type PutExecutionsRequest struct { unknownFields protoimpl.UnknownFields Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + // FieldMask for executions in the PUT update + // If `execution.id` is not specified, it means a new execution will be + // created and `update_mask` will not be applied to the creation. + // If `update_mask` is empty, update the executions as a whole. + // If `update_mask` is not empty, only update fields or properties specified + // in `update_mask`. + // Example request protos: + // 1. Add a <'key', 'val'> pair into `custom_properties`: + // { + // executions { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // string_value: "val" + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 2. Set `last_known_state` field: + // { + // executions { + // id: 1234 + // type_id: 5678 + // last_known_state: CACHED + // } + // update_mask { + // paths: "last_known_state" + // } + // } + // Please refer to `PutArtifactsRequest` for more details. + UpdateMask *fieldmaskpb.FieldMask `protobuf:"bytes,3,opt,name=update_mask,json=updateMask" json:"update_mask,omitempty"` } func (x *PutExecutionsRequest) Reset() { @@ -588,6 +736,20 @@ func (x *PutExecutionsRequest) GetExecutions() []*Execution { return nil } +func (x *PutExecutionsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +func (x *PutExecutionsRequest) GetUpdateMask() *fieldmaskpb.FieldMask { + if x != nil { + return x.UpdateMask + } + return nil +} + type PutExecutionsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -661,6 +823,8 @@ type PutExecutionTypeRequest struct { CanDeleteFields *bool `protobuf:"varint,3,opt,name=can_delete_fields,json=canDeleteFields" json:"can_delete_fields,omitempty"` // Deprecated: Do not use. AllFieldsMatch *bool `protobuf:"varint,4,opt,name=all_fields_match,json=allFieldsMatch,def=1" json:"all_fields_match,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,6,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } // Default values for PutExecutionTypeRequest fields. @@ -737,6 +901,13 @@ func (x *PutExecutionTypeRequest) GetAllFieldsMatch() bool { return Default_PutExecutionTypeRequest_AllFieldsMatch } +func (x *PutExecutionTypeRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutExecutionTypeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -791,6 +962,8 @@ type PutEventsRequest struct { unknownFields protoimpl.UnknownFields Events []*Event `protobuf:"bytes,1,rep,name=events" json:"events,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } func (x *PutEventsRequest) Reset() { @@ -832,6 +1005,13 @@ func (x *PutEventsRequest) GetEvents() []*Event { return nil } +func (x *PutEventsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutEventsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -888,6 +1068,8 @@ type PutExecutionRequest struct { Contexts []*Context `protobuf:"bytes,3,rep,name=contexts" json:"contexts,omitempty"` // Additional options to change the behavior of the method. Options *PutExecutionRequest_Options `protobuf:"bytes,4,opt,name=options" json:"options,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,5,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } func (x *PutExecutionRequest) Reset() { @@ -950,6 +1132,13 @@ func (x *PutExecutionRequest) GetOptions() *PutExecutionRequest_Options { return nil } +func (x *PutExecutionRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutExecutionResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1018,6 +1207,159 @@ func (x *PutExecutionResponse) GetContextIds() []int64 { return nil } +type PutLineageSubgraphRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` + Artifacts []*Artifact `protobuf:"bytes,2,rep,name=artifacts" json:"artifacts,omitempty"` + Contexts []*Context `protobuf:"bytes,3,rep,name=contexts" json:"contexts,omitempty"` + EventEdges []*PutLineageSubgraphRequest_EventEdge `protobuf:"bytes,4,rep,name=event_edges,json=eventEdges" json:"event_edges,omitempty"` + Options *PutLineageSubgraphRequest_Options `protobuf:"bytes,5,opt,name=options" json:"options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,6,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *PutLineageSubgraphRequest) Reset() { + *x = PutLineageSubgraphRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutLineageSubgraphRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutLineageSubgraphRequest) ProtoMessage() {} + +func (x *PutLineageSubgraphRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutLineageSubgraphRequest.ProtoReflect.Descriptor instead. +func (*PutLineageSubgraphRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{16} +} + +func (x *PutLineageSubgraphRequest) GetExecutions() []*Execution { + if x != nil { + return x.Executions + } + return nil +} + +func (x *PutLineageSubgraphRequest) GetArtifacts() []*Artifact { + if x != nil { + return x.Artifacts + } + return nil +} + +func (x *PutLineageSubgraphRequest) GetContexts() []*Context { + if x != nil { + return x.Contexts + } + return nil +} + +func (x *PutLineageSubgraphRequest) GetEventEdges() []*PutLineageSubgraphRequest_EventEdge { + if x != nil { + return x.EventEdges + } + return nil +} + +func (x *PutLineageSubgraphRequest) GetOptions() *PutLineageSubgraphRequest_Options { + if x != nil { + return x.Options + } + return nil +} + +func (x *PutLineageSubgraphRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type PutLineageSubgraphResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // A list of execution ids index-aligned with `executions` in the request + ExecutionIds []int64 `protobuf:"varint,1,rep,packed,name=execution_ids,json=executionIds" json:"execution_ids,omitempty"` + // A list of artifact ids index-aligned with `artifacts` in the request + ArtifactIds []int64 `protobuf:"varint,2,rep,packed,name=artifact_ids,json=artifactIds" json:"artifact_ids,omitempty"` + // A list of context ids index-aligned with `contexts` in the request + ContextIds []int64 `protobuf:"varint,3,rep,packed,name=context_ids,json=contextIds" json:"context_ids,omitempty"` +} + +func (x *PutLineageSubgraphResponse) Reset() { + *x = PutLineageSubgraphResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutLineageSubgraphResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutLineageSubgraphResponse) ProtoMessage() {} + +func (x *PutLineageSubgraphResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutLineageSubgraphResponse.ProtoReflect.Descriptor instead. +func (*PutLineageSubgraphResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{17} +} + +func (x *PutLineageSubgraphResponse) GetExecutionIds() []int64 { + if x != nil { + return x.ExecutionIds + } + return nil +} + +func (x *PutLineageSubgraphResponse) GetArtifactIds() []int64 { + if x != nil { + return x.ArtifactIds + } + return nil +} + +func (x *PutLineageSubgraphResponse) GetContextIds() []int64 { + if x != nil { + return x.ContextIds + } + return nil +} + type PutTypesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1041,6 +1383,8 @@ type PutTypesRequest struct { CanDeleteFields *bool `protobuf:"varint,5,opt,name=can_delete_fields,json=canDeleteFields" json:"can_delete_fields,omitempty"` // Deprecated: Do not use. AllFieldsMatch *bool `protobuf:"varint,6,opt,name=all_fields_match,json=allFieldsMatch,def=1" json:"all_fields_match,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,8,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } // Default values for PutTypesRequest fields. @@ -1051,7 +1395,7 @@ const ( func (x *PutTypesRequest) Reset() { *x = PutTypesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[16] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1064,7 +1408,7 @@ func (x *PutTypesRequest) String() string { func (*PutTypesRequest) ProtoMessage() {} func (x *PutTypesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[16] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1077,7 +1421,7 @@ func (x *PutTypesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PutTypesRequest.ProtoReflect.Descriptor instead. func (*PutTypesRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{16} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{18} } func (x *PutTypesRequest) GetArtifactTypes() []*ArtifactType { @@ -1131,6 +1475,13 @@ func (x *PutTypesRequest) GetAllFieldsMatch() bool { return Default_PutTypesRequest_AllFieldsMatch } +func (x *PutTypesRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutTypesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1147,7 +1498,7 @@ type PutTypesResponse struct { func (x *PutTypesResponse) Reset() { *x = PutTypesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[17] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1160,7 +1511,7 @@ func (x *PutTypesResponse) String() string { func (*PutTypesResponse) ProtoMessage() {} func (x *PutTypesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[17] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1173,7 +1524,7 @@ func (x *PutTypesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PutTypesResponse.ProtoReflect.Descriptor instead. func (*PutTypesResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{17} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{19} } func (x *PutTypesResponse) GetArtifactTypeIds() []int64 { @@ -1222,6 +1573,8 @@ type PutContextTypeRequest struct { CanDeleteFields *bool `protobuf:"varint,3,opt,name=can_delete_fields,json=canDeleteFields" json:"can_delete_fields,omitempty"` // Deprecated: Do not use. AllFieldsMatch *bool `protobuf:"varint,4,opt,name=all_fields_match,json=allFieldsMatch,def=1" json:"all_fields_match,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,6,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } // Default values for PutContextTypeRequest fields. @@ -1232,7 +1585,7 @@ const ( func (x *PutContextTypeRequest) Reset() { *x = PutContextTypeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[18] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1245,7 +1598,7 @@ func (x *PutContextTypeRequest) String() string { func (*PutContextTypeRequest) ProtoMessage() {} func (x *PutContextTypeRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[18] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1258,7 +1611,7 @@ func (x *PutContextTypeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PutContextTypeRequest.ProtoReflect.Descriptor instead. func (*PutContextTypeRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{18} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{20} } func (x *PutContextTypeRequest) GetContextType() *ContextType { @@ -1298,6 +1651,13 @@ func (x *PutContextTypeRequest) GetAllFieldsMatch() bool { return Default_PutContextTypeRequest_AllFieldsMatch } +func (x *PutContextTypeRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutContextTypeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1310,7 +1670,7 @@ type PutContextTypeResponse struct { func (x *PutContextTypeResponse) Reset() { *x = PutContextTypeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[19] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1323,7 +1683,7 @@ func (x *PutContextTypeResponse) String() string { func (*PutContextTypeResponse) ProtoMessage() {} func (x *PutContextTypeResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[19] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1336,7 +1696,7 @@ func (x *PutContextTypeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PutContextTypeResponse.ProtoReflect.Descriptor instead. func (*PutContextTypeResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{19} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{21} } func (x *PutContextTypeResponse) GetTypeId() int64 { @@ -1352,12 +1712,50 @@ type PutContextsRequest struct { unknownFields protoimpl.UnknownFields Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + // FieldMask for contexts in the PUT update + // If `context.id` is not specified, it means a new context will be + // created and `update_mask` will not be applied to the creation. + // If `update_mask` is empty, update the contexts as a whole. + // If `update_mask` is not empty, only update fields or properties specified + // in `update_mask`. + // Example request protos: + // 1. Add a <'key', 'val'> pair into `custom_properties`: + // { + // contexts { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // string_value: "val" + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 2. Set `name` field: + // { + // contexts { + // id: 1234 + // type_id: 5678 + // name: "set_name" + // } + // update_mask { + // paths: "name" + // } + // } + // Please refer to `PutArtifactsRequest` for more details. + UpdateMask *fieldmaskpb.FieldMask `protobuf:"bytes,3,opt,name=update_mask,json=updateMask" json:"update_mask,omitempty"` } func (x *PutContextsRequest) Reset() { *x = PutContextsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[20] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1370,7 +1768,7 @@ func (x *PutContextsRequest) String() string { func (*PutContextsRequest) ProtoMessage() {} func (x *PutContextsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[20] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1383,7 +1781,7 @@ func (x *PutContextsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PutContextsRequest.ProtoReflect.Descriptor instead. func (*PutContextsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{20} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{22} } func (x *PutContextsRequest) GetContexts() []*Context { @@ -1393,6 +1791,20 @@ func (x *PutContextsRequest) GetContexts() []*Context { return nil } +func (x *PutContextsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +func (x *PutContextsRequest) GetUpdateMask() *fieldmaskpb.FieldMask { + if x != nil { + return x.UpdateMask + } + return nil +} + type PutContextsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1405,7 +1817,7 @@ type PutContextsResponse struct { func (x *PutContextsResponse) Reset() { *x = PutContextsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[21] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1418,7 +1830,7 @@ func (x *PutContextsResponse) String() string { func (*PutContextsResponse) ProtoMessage() {} func (x *PutContextsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[21] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1431,7 +1843,7 @@ func (x *PutContextsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PutContextsResponse.ProtoReflect.Descriptor instead. func (*PutContextsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{21} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{23} } func (x *PutContextsResponse) GetContextIds() []int64 { @@ -1448,12 +1860,14 @@ type PutAttributionsAndAssociationsRequest struct { Attributions []*Attribution `protobuf:"bytes,1,rep,name=attributions" json:"attributions,omitempty"` Associations []*Association `protobuf:"bytes,2,rep,name=associations" json:"associations,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } func (x *PutAttributionsAndAssociationsRequest) Reset() { *x = PutAttributionsAndAssociationsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[22] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1466,7 +1880,7 @@ func (x *PutAttributionsAndAssociationsRequest) String() string { func (*PutAttributionsAndAssociationsRequest) ProtoMessage() {} func (x *PutAttributionsAndAssociationsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[22] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1479,7 +1893,7 @@ func (x *PutAttributionsAndAssociationsRequest) ProtoReflect() protoreflect.Mess // Deprecated: Use PutAttributionsAndAssociationsRequest.ProtoReflect.Descriptor instead. func (*PutAttributionsAndAssociationsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{22} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{24} } func (x *PutAttributionsAndAssociationsRequest) GetAttributions() []*Attribution { @@ -1496,6 +1910,13 @@ func (x *PutAttributionsAndAssociationsRequest) GetAssociations() []*Association return nil } +func (x *PutAttributionsAndAssociationsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutAttributionsAndAssociationsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1505,7 +1926,7 @@ type PutAttributionsAndAssociationsResponse struct { func (x *PutAttributionsAndAssociationsResponse) Reset() { *x = PutAttributionsAndAssociationsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[23] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1518,7 +1939,7 @@ func (x *PutAttributionsAndAssociationsResponse) String() string { func (*PutAttributionsAndAssociationsResponse) ProtoMessage() {} func (x *PutAttributionsAndAssociationsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[23] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1531,7 +1952,7 @@ func (x *PutAttributionsAndAssociationsResponse) ProtoReflect() protoreflect.Mes // Deprecated: Use PutAttributionsAndAssociationsResponse.ProtoReflect.Descriptor instead. func (*PutAttributionsAndAssociationsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{23} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{25} } type PutParentContextsRequest struct { @@ -1540,12 +1961,14 @@ type PutParentContextsRequest struct { unknownFields protoimpl.UnknownFields ParentContexts []*ParentContext `protobuf:"bytes,1,rep,name=parent_contexts,json=parentContexts" json:"parent_contexts,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } func (x *PutParentContextsRequest) Reset() { *x = PutParentContextsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[24] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1558,7 +1981,7 @@ func (x *PutParentContextsRequest) String() string { func (*PutParentContextsRequest) ProtoMessage() {} func (x *PutParentContextsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[24] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1571,7 +1994,7 @@ func (x *PutParentContextsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PutParentContextsRequest.ProtoReflect.Descriptor instead. func (*PutParentContextsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{24} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{26} } func (x *PutParentContextsRequest) GetParentContexts() []*ParentContext { @@ -1581,6 +2004,13 @@ func (x *PutParentContextsRequest) GetParentContexts() []*ParentContext { return nil } +func (x *PutParentContextsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + type PutParentContextsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1590,7 +2020,7 @@ type PutParentContextsResponse struct { func (x *PutParentContextsResponse) Reset() { *x = PutParentContextsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[25] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1603,7 +2033,7 @@ func (x *PutParentContextsResponse) String() string { func (*PutParentContextsResponse) ProtoMessage() {} func (x *PutParentContextsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[25] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1616,7 +2046,7 @@ func (x *PutParentContextsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PutParentContextsResponse.ProtoReflect.Descriptor instead. func (*PutParentContextsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{25} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{27} } type GetArtifactsByTypeRequest struct { @@ -1631,6 +2061,12 @@ type GetArtifactsByTypeRequest struct { // Currently supports: // 1. Field to order the results. // 2. Page size. + // If set, the request will + // first fetch all artifacts with specified `type_name` and `type_version`, + // then order by a specifield field + // finally find the correct page and return #Artifacts of the page size. + // Higher-level APIs may only use the functionalies partially. + // Please reference the API documentation for the API behaviors. Options *ListOperationOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` @@ -1639,7 +2075,7 @@ type GetArtifactsByTypeRequest struct { func (x *GetArtifactsByTypeRequest) Reset() { *x = GetArtifactsByTypeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[26] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1652,7 +2088,7 @@ func (x *GetArtifactsByTypeRequest) String() string { func (*GetArtifactsByTypeRequest) ProtoMessage() {} func (x *GetArtifactsByTypeRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[26] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1665,7 +2101,7 @@ func (x *GetArtifactsByTypeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsByTypeRequest.ProtoReflect.Descriptor instead. func (*GetArtifactsByTypeRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{26} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{28} } func (x *GetArtifactsByTypeRequest) GetTypeName() string { @@ -1710,7 +2146,7 @@ type GetArtifactsByTypeResponse struct { func (x *GetArtifactsByTypeResponse) Reset() { *x = GetArtifactsByTypeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[27] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1723,7 +2159,7 @@ func (x *GetArtifactsByTypeResponse) String() string { func (*GetArtifactsByTypeResponse) ProtoMessage() {} func (x *GetArtifactsByTypeResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[27] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1736,7 +2172,7 @@ func (x *GetArtifactsByTypeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsByTypeResponse.ProtoReflect.Descriptor instead. func (*GetArtifactsByTypeResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{27} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{29} } func (x *GetArtifactsByTypeResponse) GetArtifacts() []*Artifact { @@ -1770,7 +2206,7 @@ type GetArtifactByTypeAndNameRequest struct { func (x *GetArtifactByTypeAndNameRequest) Reset() { *x = GetArtifactByTypeAndNameRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[28] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1783,7 +2219,7 @@ func (x *GetArtifactByTypeAndNameRequest) String() string { func (*GetArtifactByTypeAndNameRequest) ProtoMessage() {} func (x *GetArtifactByTypeAndNameRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[28] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1796,7 +2232,7 @@ func (x *GetArtifactByTypeAndNameRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactByTypeAndNameRequest.ProtoReflect.Descriptor instead. func (*GetArtifactByTypeAndNameRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{28} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{30} } func (x *GetArtifactByTypeAndNameRequest) GetTypeName() string { @@ -1838,7 +2274,7 @@ type GetArtifactByTypeAndNameResponse struct { func (x *GetArtifactByTypeAndNameResponse) Reset() { *x = GetArtifactByTypeAndNameResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[29] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1851,7 +2287,7 @@ func (x *GetArtifactByTypeAndNameResponse) String() string { func (*GetArtifactByTypeAndNameResponse) ProtoMessage() {} func (x *GetArtifactByTypeAndNameResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[29] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1864,7 +2300,7 @@ func (x *GetArtifactByTypeAndNameResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactByTypeAndNameResponse.ProtoReflect.Descriptor instead. func (*GetArtifactByTypeAndNameResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{29} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{31} } func (x *GetArtifactByTypeAndNameResponse) GetArtifact() *Artifact { @@ -1881,14 +2317,31 @@ type GetArtifactsByIDRequest struct { // A list of artifact ids to retrieve. ArtifactIds []int64 `protobuf:"varint,1,rep,name=artifact_ids,json=artifactIds" json:"artifact_ids,omitempty"` + // An option to populate all the ArtifactTypes in the response. + // If true, returns retrieved Artifacts and their artifact types, which can be + // matched by type_ids. + // If false, returns only the retrieved Artifacts. + // Example request proto: + // { + // artifact_ids: 101, + // populate_artifact_types: true, + // } + // The response will contain an artifact with id = 101 and an artifact type + // with id = artifact.type_id(). + PopulateArtifactTypes *bool `protobuf:"varint,3,opt,name=populate_artifact_types,json=populateArtifactTypes,def=0" json:"populate_artifact_types,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } +// Default values for GetArtifactsByIDRequest fields. +const ( + Default_GetArtifactsByIDRequest_PopulateArtifactTypes = bool(false) +) + func (x *GetArtifactsByIDRequest) Reset() { *x = GetArtifactsByIDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[30] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1901,7 +2354,7 @@ func (x *GetArtifactsByIDRequest) String() string { func (*GetArtifactsByIDRequest) ProtoMessage() {} func (x *GetArtifactsByIDRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[30] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1914,7 +2367,7 @@ func (x *GetArtifactsByIDRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsByIDRequest.ProtoReflect.Descriptor instead. func (*GetArtifactsByIDRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{30} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{32} } func (x *GetArtifactsByIDRequest) GetArtifactIds() []int64 { @@ -1924,6 +2377,13 @@ func (x *GetArtifactsByIDRequest) GetArtifactIds() []int64 { return nil } +func (x *GetArtifactsByIDRequest) GetPopulateArtifactTypes() bool { + if x != nil && x.PopulateArtifactTypes != nil { + return *x.PopulateArtifactTypes + } + return Default_GetArtifactsByIDRequest_PopulateArtifactTypes +} + func (x *GetArtifactsByIDRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions @@ -1937,14 +2397,17 @@ type GetArtifactsByIDResponse struct { unknownFields protoimpl.UnknownFields // Artifacts with matching ids. - // This is not index-aligned: if an id is not found, it is not + // This is not index-aligned: if an id is not found, it is not returned. Artifacts []*Artifact `protobuf:"bytes,1,rep,name=artifacts" json:"artifacts,omitempty"` + // ArtifactTypes populated with matching type_ids owned by `artifacts`. + // This is not index-aligned: if a type_id is not found, it is not returned. + ArtifactTypes []*ArtifactType `protobuf:"bytes,2,rep,name=artifact_types,json=artifactTypes" json:"artifact_types,omitempty"` } func (x *GetArtifactsByIDResponse) Reset() { *x = GetArtifactsByIDResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[31] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1957,7 +2420,7 @@ func (x *GetArtifactsByIDResponse) String() string { func (*GetArtifactsByIDResponse) ProtoMessage() {} func (x *GetArtifactsByIDResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[31] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1970,7 +2433,7 @@ func (x *GetArtifactsByIDResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsByIDResponse.ProtoReflect.Descriptor instead. func (*GetArtifactsByIDResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{31} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{33} } func (x *GetArtifactsByIDResponse) GetArtifacts() []*Artifact { @@ -1980,6 +2443,13 @@ func (x *GetArtifactsByIDResponse) GetArtifacts() []*Artifact { return nil } +func (x *GetArtifactsByIDResponse) GetArtifactTypes() []*ArtifactType { + if x != nil { + return x.ArtifactTypes + } + return nil +} + // Request to retrieve Artifacts using List options. // If option is not specified then all Artifacts are returned. type GetArtifactsRequest struct { @@ -1988,9 +2458,8 @@ type GetArtifactsRequest struct { unknownFields protoimpl.UnknownFields // Specify options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. + // Please refer to the documentation of ListOperationOptions for the supported + // functionalities. Options *ListOperationOptions `protobuf:"bytes,1,opt,name=options" json:"options,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` @@ -1999,7 +2468,7 @@ type GetArtifactsRequest struct { func (x *GetArtifactsRequest) Reset() { *x = GetArtifactsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[32] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2012,7 +2481,7 @@ func (x *GetArtifactsRequest) String() string { func (*GetArtifactsRequest) ProtoMessage() {} func (x *GetArtifactsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[32] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2025,7 +2494,7 @@ func (x *GetArtifactsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsRequest.ProtoReflect.Descriptor instead. func (*GetArtifactsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{32} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{34} } func (x *GetArtifactsRequest) GetOptions() *ListOperationOptions { @@ -2057,7 +2526,7 @@ type GetArtifactsResponse struct { func (x *GetArtifactsResponse) Reset() { *x = GetArtifactsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[33] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2070,7 +2539,7 @@ func (x *GetArtifactsResponse) String() string { func (*GetArtifactsResponse) ProtoMessage() {} func (x *GetArtifactsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[33] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2083,7 +2552,7 @@ func (x *GetArtifactsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsResponse.ProtoReflect.Descriptor instead. func (*GetArtifactsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{33} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{35} } func (x *GetArtifactsResponse) GetArtifacts() []*Artifact { @@ -2114,7 +2583,7 @@ type GetArtifactsByURIRequest struct { func (x *GetArtifactsByURIRequest) Reset() { *x = GetArtifactsByURIRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[34] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2127,7 +2596,7 @@ func (x *GetArtifactsByURIRequest) String() string { func (*GetArtifactsByURIRequest) ProtoMessage() {} func (x *GetArtifactsByURIRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[34] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2140,7 +2609,7 @@ func (x *GetArtifactsByURIRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsByURIRequest.ProtoReflect.Descriptor instead. func (*GetArtifactsByURIRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{34} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{36} } func (x *GetArtifactsByURIRequest) GetUris() []string { @@ -2168,7 +2637,7 @@ type GetArtifactsByURIResponse struct { func (x *GetArtifactsByURIResponse) Reset() { *x = GetArtifactsByURIResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[35] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2181,7 +2650,7 @@ func (x *GetArtifactsByURIResponse) String() string { func (*GetArtifactsByURIResponse) ProtoMessage() {} func (x *GetArtifactsByURIResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[35] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2194,7 +2663,7 @@ func (x *GetArtifactsByURIResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactsByURIResponse.ProtoReflect.Descriptor instead. func (*GetArtifactsByURIResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{35} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{37} } func (x *GetArtifactsByURIResponse) GetArtifacts() []*Artifact { @@ -2212,9 +2681,8 @@ type GetExecutionsRequest struct { unknownFields protoimpl.UnknownFields // Specify options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. + // Please refer to the documentation of ListOperationOptions for the supported + // functionalities. Options *ListOperationOptions `protobuf:"bytes,1,opt,name=options" json:"options,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` @@ -2223,7 +2691,7 @@ type GetExecutionsRequest struct { func (x *GetExecutionsRequest) Reset() { *x = GetExecutionsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[36] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2236,7 +2704,7 @@ func (x *GetExecutionsRequest) String() string { func (*GetExecutionsRequest) ProtoMessage() {} func (x *GetExecutionsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[36] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2249,7 +2717,7 @@ func (x *GetExecutionsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetExecutionsRequest.ProtoReflect.Descriptor instead. func (*GetExecutionsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{36} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{38} } func (x *GetExecutionsRequest) GetOptions() *ListOperationOptions { @@ -2281,7 +2749,7 @@ type GetExecutionsResponse struct { func (x *GetExecutionsResponse) Reset() { *x = GetExecutionsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[37] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2294,7 +2762,7 @@ func (x *GetExecutionsResponse) String() string { func (*GetExecutionsResponse) ProtoMessage() {} func (x *GetExecutionsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[37] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2307,7 +2775,7 @@ func (x *GetExecutionsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetExecutionsResponse.ProtoReflect.Descriptor instead. func (*GetExecutionsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{37} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{39} } func (x *GetExecutionsResponse) GetExecutions() []*Execution { @@ -2339,7 +2807,7 @@ type GetArtifactTypeRequest struct { func (x *GetArtifactTypeRequest) Reset() { *x = GetArtifactTypeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[38] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2352,7 +2820,7 @@ func (x *GetArtifactTypeRequest) String() string { func (*GetArtifactTypeRequest) ProtoMessage() {} func (x *GetArtifactTypeRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[38] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2365,7 +2833,7 @@ func (x *GetArtifactTypeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactTypeRequest.ProtoReflect.Descriptor instead. func (*GetArtifactTypeRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{38} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{40} } func (x *GetArtifactTypeRequest) GetTypeName() string { @@ -2401,7 +2869,7 @@ type GetArtifactTypeResponse struct { func (x *GetArtifactTypeResponse) Reset() { *x = GetArtifactTypeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[39] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2414,7 +2882,7 @@ func (x *GetArtifactTypeResponse) String() string { func (*GetArtifactTypeResponse) ProtoMessage() {} func (x *GetArtifactTypeResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[39] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2427,7 +2895,7 @@ func (x *GetArtifactTypeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactTypeResponse.ProtoReflect.Descriptor instead. func (*GetArtifactTypeResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{39} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{41} } func (x *GetArtifactTypeResponse) GetArtifactType() *ArtifactType { @@ -2449,7 +2917,7 @@ type GetArtifactTypesRequest struct { func (x *GetArtifactTypesRequest) Reset() { *x = GetArtifactTypesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[40] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2462,7 +2930,7 @@ func (x *GetArtifactTypesRequest) String() string { func (*GetArtifactTypesRequest) ProtoMessage() {} func (x *GetArtifactTypesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[40] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2475,7 +2943,7 @@ func (x *GetArtifactTypesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactTypesRequest.ProtoReflect.Descriptor instead. func (*GetArtifactTypesRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{40} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{42} } func (x *GetArtifactTypesRequest) GetTransactionOptions() *TransactionOptions { @@ -2496,7 +2964,7 @@ type GetArtifactTypesResponse struct { func (x *GetArtifactTypesResponse) Reset() { *x = GetArtifactTypesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[41] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2509,7 +2977,7 @@ func (x *GetArtifactTypesResponse) String() string { func (*GetArtifactTypesResponse) ProtoMessage() {} func (x *GetArtifactTypesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[41] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2522,7 +2990,7 @@ func (x *GetArtifactTypesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetArtifactTypesResponse.ProtoReflect.Descriptor instead. func (*GetArtifactTypesResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{41} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{43} } func (x *GetArtifactTypesResponse) GetArtifactTypes() []*ArtifactType { @@ -2544,7 +3012,7 @@ type GetExecutionTypesRequest struct { func (x *GetExecutionTypesRequest) Reset() { *x = GetExecutionTypesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[42] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2557,7 +3025,7 @@ func (x *GetExecutionTypesRequest) String() string { func (*GetExecutionTypesRequest) ProtoMessage() {} func (x *GetExecutionTypesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[42] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2570,7 +3038,7 @@ func (x *GetExecutionTypesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetExecutionTypesRequest.ProtoReflect.Descriptor instead. func (*GetExecutionTypesRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{42} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{44} } func (x *GetExecutionTypesRequest) GetTransactionOptions() *TransactionOptions { @@ -2591,7 +3059,7 @@ type GetExecutionTypesResponse struct { func (x *GetExecutionTypesResponse) Reset() { *x = GetExecutionTypesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[43] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2604,7 +3072,7 @@ func (x *GetExecutionTypesResponse) String() string { func (*GetExecutionTypesResponse) ProtoMessage() {} func (x *GetExecutionTypesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[43] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2617,7 +3085,7 @@ func (x *GetExecutionTypesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetExecutionTypesResponse.ProtoReflect.Descriptor instead. func (*GetExecutionTypesResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{43} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{45} } func (x *GetExecutionTypesResponse) GetExecutionTypes() []*ExecutionType { @@ -2639,7 +3107,7 @@ type GetContextTypesRequest struct { func (x *GetContextTypesRequest) Reset() { *x = GetContextTypesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[44] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2652,7 +3120,7 @@ func (x *GetContextTypesRequest) String() string { func (*GetContextTypesRequest) ProtoMessage() {} func (x *GetContextTypesRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[44] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2665,7 +3133,7 @@ func (x *GetContextTypesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetContextTypesRequest.ProtoReflect.Descriptor instead. func (*GetContextTypesRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{44} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{46} } func (x *GetContextTypesRequest) GetTransactionOptions() *TransactionOptions { @@ -2686,7 +3154,7 @@ type GetContextTypesResponse struct { func (x *GetContextTypesResponse) Reset() { *x = GetContextTypesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[45] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2699,7 +3167,7 @@ func (x *GetContextTypesResponse) String() string { func (*GetContextTypesResponse) ProtoMessage() {} func (x *GetContextTypesResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[45] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2712,7 +3180,7 @@ func (x *GetContextTypesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetContextTypesResponse.ProtoReflect.Descriptor instead. func (*GetContextTypesResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{45} + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{47} } func (x *GetContextTypesResponse) GetContextTypes() []*ContextType { @@ -2722,40 +3190,33 @@ func (x *GetContextTypesResponse) GetContextTypes() []*ContextType { return nil } -type GetExecutionsByTypeRequest struct { +type GetArtifactsByExternalIdsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` - // If not set, it looks for the type with type_name with default type_version. - TypeVersion *string `protobuf:"bytes,2,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` - // Specify List options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. - Options *ListOperationOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"` + ExternalIds []string `protobuf:"bytes,1,rep,name=external_ids,json=externalIds" json:"external_ids,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetExecutionsByTypeRequest) Reset() { - *x = GetExecutionsByTypeRequest{} +func (x *GetArtifactsByExternalIdsRequest) Reset() { + *x = GetArtifactsByExternalIdsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[46] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionsByTypeRequest) String() string { +func (x *GetArtifactsByExternalIdsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionsByTypeRequest) ProtoMessage() {} +func (*GetArtifactsByExternalIdsRequest) ProtoMessage() {} -func (x *GetExecutionsByTypeRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[46] +func (x *GetArtifactsByExternalIdsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2766,67 +3227,50 @@ func (x *GetExecutionsByTypeRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionsByTypeRequest.ProtoReflect.Descriptor instead. -func (*GetExecutionsByTypeRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{46} -} - -func (x *GetExecutionsByTypeRequest) GetTypeName() string { - if x != nil && x.TypeName != nil { - return *x.TypeName - } - return "" -} - -func (x *GetExecutionsByTypeRequest) GetTypeVersion() string { - if x != nil && x.TypeVersion != nil { - return *x.TypeVersion - } - return "" +// Deprecated: Use GetArtifactsByExternalIdsRequest.ProtoReflect.Descriptor instead. +func (*GetArtifactsByExternalIdsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{48} } -func (x *GetExecutionsByTypeRequest) GetOptions() *ListOperationOptions { +func (x *GetArtifactsByExternalIdsRequest) GetExternalIds() []string { if x != nil { - return x.Options + return x.ExternalIds } return nil } -func (x *GetExecutionsByTypeRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetArtifactsByExternalIdsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetExecutionsByTypeResponse struct { +type GetArtifactsByExternalIdsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` - // Token to use to retrieve next page of results if list options are used in - // the request. - NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` + Artifacts []*Artifact `protobuf:"bytes,1,rep,name=artifacts" json:"artifacts,omitempty"` } -func (x *GetExecutionsByTypeResponse) Reset() { - *x = GetExecutionsByTypeResponse{} +func (x *GetArtifactsByExternalIdsResponse) Reset() { + *x = GetArtifactsByExternalIdsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[47] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionsByTypeResponse) String() string { +func (x *GetArtifactsByExternalIdsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionsByTypeResponse) ProtoMessage() {} +func (*GetArtifactsByExternalIdsResponse) ProtoMessage() {} -func (x *GetExecutionsByTypeResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[47] +func (x *GetArtifactsByExternalIdsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2837,56 +3281,45 @@ func (x *GetExecutionsByTypeResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionsByTypeResponse.ProtoReflect.Descriptor instead. -func (*GetExecutionsByTypeResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{47} +// Deprecated: Use GetArtifactsByExternalIdsResponse.ProtoReflect.Descriptor instead. +func (*GetArtifactsByExternalIdsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{49} } -func (x *GetExecutionsByTypeResponse) GetExecutions() []*Execution { +func (x *GetArtifactsByExternalIdsResponse) GetArtifacts() []*Artifact { if x != nil { - return x.Executions + return x.Artifacts } return nil } -func (x *GetExecutionsByTypeResponse) GetNextPageToken() string { - if x != nil && x.NextPageToken != nil { - return *x.NextPageToken - } - return "" -} - -type GetExecutionByTypeAndNameRequest struct { +type GetExecutionsByExternalIdsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` - // If not set, it looks for the type with type_name and execution_name with - // default type_version. - TypeVersion *string `protobuf:"bytes,3,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` - ExecutionName *string `protobuf:"bytes,2,opt,name=execution_name,json=executionName" json:"execution_name,omitempty"` + ExternalIds []string `protobuf:"bytes,1,rep,name=external_ids,json=externalIds" json:"external_ids,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetExecutionByTypeAndNameRequest) Reset() { - *x = GetExecutionByTypeAndNameRequest{} +func (x *GetExecutionsByExternalIdsRequest) Reset() { + *x = GetExecutionsByExternalIdsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[48] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionByTypeAndNameRequest) String() string { +func (x *GetExecutionsByExternalIdsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionByTypeAndNameRequest) ProtoMessage() {} +func (*GetExecutionsByExternalIdsRequest) ProtoMessage() {} -func (x *GetExecutionByTypeAndNameRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[48] +func (x *GetExecutionsByExternalIdsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2897,64 +3330,50 @@ func (x *GetExecutionByTypeAndNameRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionByTypeAndNameRequest.ProtoReflect.Descriptor instead. -func (*GetExecutionByTypeAndNameRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{48} +// Deprecated: Use GetExecutionsByExternalIdsRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionsByExternalIdsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{50} } -func (x *GetExecutionByTypeAndNameRequest) GetTypeName() string { - if x != nil && x.TypeName != nil { - return *x.TypeName +func (x *GetExecutionsByExternalIdsRequest) GetExternalIds() []string { + if x != nil { + return x.ExternalIds } - return "" + return nil } -func (x *GetExecutionByTypeAndNameRequest) GetTypeVersion() string { - if x != nil && x.TypeVersion != nil { - return *x.TypeVersion - } - return "" -} - -func (x *GetExecutionByTypeAndNameRequest) GetExecutionName() string { - if x != nil && x.ExecutionName != nil { - return *x.ExecutionName - } - return "" -} - -func (x *GetExecutionByTypeAndNameRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetExecutionsByExternalIdsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetExecutionByTypeAndNameResponse struct { +type GetExecutionsByExternalIdsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Execution *Execution `protobuf:"bytes,1,opt,name=execution" json:"execution,omitempty"` + Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` } -func (x *GetExecutionByTypeAndNameResponse) Reset() { - *x = GetExecutionByTypeAndNameResponse{} +func (x *GetExecutionsByExternalIdsResponse) Reset() { + *x = GetExecutionsByExternalIdsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[49] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionByTypeAndNameResponse) String() string { +func (x *GetExecutionsByExternalIdsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionByTypeAndNameResponse) ProtoMessage() {} +func (*GetExecutionsByExternalIdsResponse) ProtoMessage() {} -func (x *GetExecutionByTypeAndNameResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[49] +func (x *GetExecutionsByExternalIdsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2965,46 +3384,45 @@ func (x *GetExecutionByTypeAndNameResponse) ProtoReflect() protoreflect.Message return mi.MessageOf(x) } -// Deprecated: Use GetExecutionByTypeAndNameResponse.ProtoReflect.Descriptor instead. -func (*GetExecutionByTypeAndNameResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{49} +// Deprecated: Use GetExecutionsByExternalIdsResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionsByExternalIdsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{51} } -func (x *GetExecutionByTypeAndNameResponse) GetExecution() *Execution { +func (x *GetExecutionsByExternalIdsResponse) GetExecutions() []*Execution { if x != nil { - return x.Execution + return x.Executions } return nil } -type GetExecutionsByIDRequest struct { +type GetContextsByExternalIdsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // A list of execution ids to retrieve. - ExecutionIds []int64 `protobuf:"varint,1,rep,name=execution_ids,json=executionIds" json:"execution_ids,omitempty"` + ExternalIds []string `protobuf:"bytes,1,rep,name=external_ids,json=externalIds" json:"external_ids,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetExecutionsByIDRequest) Reset() { - *x = GetExecutionsByIDRequest{} +func (x *GetContextsByExternalIdsRequest) Reset() { + *x = GetContextsByExternalIdsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[50] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionsByIDRequest) String() string { +func (x *GetContextsByExternalIdsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionsByIDRequest) ProtoMessage() {} +func (*GetContextsByExternalIdsRequest) ProtoMessage() {} -func (x *GetExecutionsByIDRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[50] +func (x *GetContextsByExternalIdsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3015,52 +3433,50 @@ func (x *GetExecutionsByIDRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionsByIDRequest.ProtoReflect.Descriptor instead. -func (*GetExecutionsByIDRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{50} +// Deprecated: Use GetContextsByExternalIdsRequest.ProtoReflect.Descriptor instead. +func (*GetContextsByExternalIdsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{52} } -func (x *GetExecutionsByIDRequest) GetExecutionIds() []int64 { +func (x *GetContextsByExternalIdsRequest) GetExternalIds() []string { if x != nil { - return x.ExecutionIds + return x.ExternalIds } return nil } -func (x *GetExecutionsByIDRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextsByExternalIdsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetExecutionsByIDResponse struct { +type GetContextsByExternalIdsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The result is not index-aligned: if an id is not found, it is not - // returned. - Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` + Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` } -func (x *GetExecutionsByIDResponse) Reset() { - *x = GetExecutionsByIDResponse{} +func (x *GetContextsByExternalIdsResponse) Reset() { + *x = GetContextsByExternalIdsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[51] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionsByIDResponse) String() string { +func (x *GetContextsByExternalIdsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionsByIDResponse) ProtoMessage() {} +func (*GetContextsByExternalIdsResponse) ProtoMessage() {} -func (x *GetExecutionsByIDResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[51] +func (x *GetContextsByExternalIdsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3071,47 +3487,45 @@ func (x *GetExecutionsByIDResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionsByIDResponse.ProtoReflect.Descriptor instead. -func (*GetExecutionsByIDResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{51} +// Deprecated: Use GetContextsByExternalIdsResponse.ProtoReflect.Descriptor instead. +func (*GetContextsByExternalIdsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{53} } -func (x *GetExecutionsByIDResponse) GetExecutions() []*Execution { +func (x *GetContextsByExternalIdsResponse) GetContexts() []*Context { if x != nil { - return x.Executions + return x.Contexts } return nil } -type GetExecutionTypeRequest struct { +type GetArtifactTypesByExternalIdsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` - // If not set, it looks for the type with type_name with default type_version. - TypeVersion *string `protobuf:"bytes,2,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` + ExternalIds []string `protobuf:"bytes,1,rep,name=external_ids,json=externalIds" json:"external_ids,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetExecutionTypeRequest) Reset() { - *x = GetExecutionTypeRequest{} +func (x *GetArtifactTypesByExternalIdsRequest) Reset() { + *x = GetArtifactTypesByExternalIdsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[52] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionTypeRequest) String() string { +func (x *GetArtifactTypesByExternalIdsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionTypeRequest) ProtoMessage() {} +func (*GetArtifactTypesByExternalIdsRequest) ProtoMessage() {} -func (x *GetExecutionTypeRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[52] +func (x *GetArtifactTypesByExternalIdsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3122,58 +3536,50 @@ func (x *GetExecutionTypeRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionTypeRequest.ProtoReflect.Descriptor instead. -func (*GetExecutionTypeRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{52} -} - -func (x *GetExecutionTypeRequest) GetTypeName() string { - if x != nil && x.TypeName != nil { - return *x.TypeName - } - return "" +// Deprecated: Use GetArtifactTypesByExternalIdsRequest.ProtoReflect.Descriptor instead. +func (*GetArtifactTypesByExternalIdsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{54} } -func (x *GetExecutionTypeRequest) GetTypeVersion() string { - if x != nil && x.TypeVersion != nil { - return *x.TypeVersion +func (x *GetArtifactTypesByExternalIdsRequest) GetExternalIds() []string { + if x != nil { + return x.ExternalIds } - return "" + return nil } -func (x *GetExecutionTypeRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetArtifactTypesByExternalIdsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetExecutionTypeResponse struct { +type GetArtifactTypesByExternalIdsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Gets an execution type, or clear if it does not exist. - ExecutionType *ExecutionType `protobuf:"bytes,1,opt,name=execution_type,json=executionType" json:"execution_type,omitempty"` + ArtifactTypes []*ArtifactType `protobuf:"bytes,1,rep,name=artifact_types,json=artifactTypes" json:"artifact_types,omitempty"` } -func (x *GetExecutionTypeResponse) Reset() { - *x = GetExecutionTypeResponse{} +func (x *GetArtifactTypesByExternalIdsResponse) Reset() { + *x = GetArtifactTypesByExternalIdsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[53] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionTypeResponse) String() string { +func (x *GetArtifactTypesByExternalIdsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionTypeResponse) ProtoMessage() {} +func (*GetArtifactTypesByExternalIdsResponse) ProtoMessage() {} -func (x *GetExecutionTypeResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[53] +func (x *GetArtifactTypesByExternalIdsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3184,46 +3590,45 @@ func (x *GetExecutionTypeResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionTypeResponse.ProtoReflect.Descriptor instead. -func (*GetExecutionTypeResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{53} +// Deprecated: Use GetArtifactTypesByExternalIdsResponse.ProtoReflect.Descriptor instead. +func (*GetArtifactTypesByExternalIdsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{55} } -func (x *GetExecutionTypeResponse) GetExecutionType() *ExecutionType { +func (x *GetArtifactTypesByExternalIdsResponse) GetArtifactTypes() []*ArtifactType { if x != nil { - return x.ExecutionType + return x.ArtifactTypes } return nil } -// Gets all events with matching execution ids. -type GetEventsByExecutionIDsRequest struct { +type GetExecutionTypesByExternalIdsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ExecutionIds []int64 `protobuf:"varint,1,rep,name=execution_ids,json=executionIds" json:"execution_ids,omitempty"` + ExternalIds []string `protobuf:"bytes,1,rep,name=external_ids,json=externalIds" json:"external_ids,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetEventsByExecutionIDsRequest) Reset() { - *x = GetEventsByExecutionIDsRequest{} +func (x *GetExecutionTypesByExternalIdsRequest) Reset() { + *x = GetExecutionTypesByExternalIdsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[54] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetEventsByExecutionIDsRequest) String() string { +func (x *GetExecutionTypesByExternalIdsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetEventsByExecutionIDsRequest) ProtoMessage() {} +func (*GetExecutionTypesByExternalIdsRequest) ProtoMessage() {} -func (x *GetEventsByExecutionIDsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[54] +func (x *GetExecutionTypesByExternalIdsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3234,50 +3639,50 @@ func (x *GetEventsByExecutionIDsRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetEventsByExecutionIDsRequest.ProtoReflect.Descriptor instead. -func (*GetEventsByExecutionIDsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{54} +// Deprecated: Use GetExecutionTypesByExternalIdsRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionTypesByExternalIdsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{56} } -func (x *GetEventsByExecutionIDsRequest) GetExecutionIds() []int64 { +func (x *GetExecutionTypesByExternalIdsRequest) GetExternalIds() []string { if x != nil { - return x.ExecutionIds + return x.ExternalIds } return nil } -func (x *GetEventsByExecutionIDsRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetExecutionTypesByExternalIdsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetEventsByExecutionIDsResponse struct { +type GetExecutionTypesByExternalIdsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Events []*Event `protobuf:"bytes,1,rep,name=events" json:"events,omitempty"` + ExecutionTypes []*ExecutionType `protobuf:"bytes,1,rep,name=execution_types,json=executionTypes" json:"execution_types,omitempty"` } -func (x *GetEventsByExecutionIDsResponse) Reset() { - *x = GetEventsByExecutionIDsResponse{} +func (x *GetExecutionTypesByExternalIdsResponse) Reset() { + *x = GetExecutionTypesByExternalIdsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[55] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetEventsByExecutionIDsResponse) String() string { +func (x *GetExecutionTypesByExternalIdsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetEventsByExecutionIDsResponse) ProtoMessage() {} +func (*GetExecutionTypesByExternalIdsResponse) ProtoMessage() {} -func (x *GetEventsByExecutionIDsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[55] +func (x *GetExecutionTypesByExternalIdsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3288,45 +3693,45 @@ func (x *GetEventsByExecutionIDsResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetEventsByExecutionIDsResponse.ProtoReflect.Descriptor instead. -func (*GetEventsByExecutionIDsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{55} +// Deprecated: Use GetExecutionTypesByExternalIdsResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionTypesByExternalIdsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{57} } -func (x *GetEventsByExecutionIDsResponse) GetEvents() []*Event { +func (x *GetExecutionTypesByExternalIdsResponse) GetExecutionTypes() []*ExecutionType { if x != nil { - return x.Events + return x.ExecutionTypes } return nil } -type GetEventsByArtifactIDsRequest struct { +type GetContextTypesByExternalIdsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ArtifactIds []int64 `protobuf:"varint,1,rep,name=artifact_ids,json=artifactIds" json:"artifact_ids,omitempty"` + ExternalIds []string `protobuf:"bytes,1,rep,name=external_ids,json=externalIds" json:"external_ids,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetEventsByArtifactIDsRequest) Reset() { - *x = GetEventsByArtifactIDsRequest{} +func (x *GetContextTypesByExternalIdsRequest) Reset() { + *x = GetContextTypesByExternalIdsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[56] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetEventsByArtifactIDsRequest) String() string { +func (x *GetContextTypesByExternalIdsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetEventsByArtifactIDsRequest) ProtoMessage() {} +func (*GetContextTypesByExternalIdsRequest) ProtoMessage() {} -func (x *GetEventsByArtifactIDsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[56] +func (x *GetContextTypesByExternalIdsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3337,50 +3742,50 @@ func (x *GetEventsByArtifactIDsRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetEventsByArtifactIDsRequest.ProtoReflect.Descriptor instead. -func (*GetEventsByArtifactIDsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{56} +// Deprecated: Use GetContextTypesByExternalIdsRequest.ProtoReflect.Descriptor instead. +func (*GetContextTypesByExternalIdsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{58} } -func (x *GetEventsByArtifactIDsRequest) GetArtifactIds() []int64 { +func (x *GetContextTypesByExternalIdsRequest) GetExternalIds() []string { if x != nil { - return x.ArtifactIds + return x.ExternalIds } return nil } -func (x *GetEventsByArtifactIDsRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextTypesByExternalIdsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetEventsByArtifactIDsResponse struct { +type GetContextTypesByExternalIdsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Events []*Event `protobuf:"bytes,1,rep,name=events" json:"events,omitempty"` + ContextTypes []*ContextType `protobuf:"bytes,1,rep,name=context_types,json=contextTypes" json:"context_types,omitempty"` } -func (x *GetEventsByArtifactIDsResponse) Reset() { - *x = GetEventsByArtifactIDsResponse{} +func (x *GetContextTypesByExternalIdsResponse) Reset() { + *x = GetContextTypesByExternalIdsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[57] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetEventsByArtifactIDsResponse) String() string { +func (x *GetContextTypesByExternalIdsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetEventsByArtifactIDsResponse) ProtoMessage() {} +func (*GetContextTypesByExternalIdsResponse) ProtoMessage() {} -func (x *GetEventsByArtifactIDsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[57] +func (x *GetContextTypesByExternalIdsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3391,45 +3796,58 @@ func (x *GetEventsByArtifactIDsResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetEventsByArtifactIDsResponse.ProtoReflect.Descriptor instead. -func (*GetEventsByArtifactIDsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{57} +// Deprecated: Use GetContextTypesByExternalIdsResponse.ProtoReflect.Descriptor instead. +func (*GetContextTypesByExternalIdsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{59} } -func (x *GetEventsByArtifactIDsResponse) GetEvents() []*Event { +func (x *GetContextTypesByExternalIdsResponse) GetContextTypes() []*ContextType { if x != nil { - return x.Events + return x.ContextTypes } return nil } -type GetArtifactTypesByIDRequest struct { +type GetExecutionsByTypeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeIds []int64 `protobuf:"varint,1,rep,name=type_ids,json=typeIds" json:"type_ids,omitempty"` + TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` + // If not set, it looks for the type with type_name with default type_version. + TypeVersion *string `protobuf:"bytes,2,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` + // Specify List options. + // Currently supports: + // 1. Field to order the results. + // 2. Page size. + // If set, the request will + // first fetch all executions with specified `type_name` and `type_version`, + // then order by a specifield field + // finally find the correct page and return #Executions of the page size. + // Higher-level APIs may only use the functionalies partially. + // Please reference the API documentation for the API behaviors. + Options *ListOperationOptions `protobuf:"bytes,3,opt,name=options" json:"options,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetArtifactTypesByIDRequest) Reset() { - *x = GetArtifactTypesByIDRequest{} +func (x *GetExecutionsByTypeRequest) Reset() { + *x = GetExecutionsByTypeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[58] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetArtifactTypesByIDRequest) String() string { +func (x *GetExecutionsByTypeRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetArtifactTypesByIDRequest) ProtoMessage() {} +func (*GetExecutionsByTypeRequest) ProtoMessage() {} -func (x *GetArtifactTypesByIDRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[58] +func (x *GetExecutionsByTypeRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3440,52 +3858,67 @@ func (x *GetArtifactTypesByIDRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetArtifactTypesByIDRequest.ProtoReflect.Descriptor instead. -func (*GetArtifactTypesByIDRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{58} +// Deprecated: Use GetExecutionsByTypeRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionsByTypeRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{60} } -func (x *GetArtifactTypesByIDRequest) GetTypeIds() []int64 { +func (x *GetExecutionsByTypeRequest) GetTypeName() string { + if x != nil && x.TypeName != nil { + return *x.TypeName + } + return "" +} + +func (x *GetExecutionsByTypeRequest) GetTypeVersion() string { + if x != nil && x.TypeVersion != nil { + return *x.TypeVersion + } + return "" +} + +func (x *GetExecutionsByTypeRequest) GetOptions() *ListOperationOptions { if x != nil { - return x.TypeIds + return x.Options } return nil } -func (x *GetArtifactTypesByIDRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetExecutionsByTypeRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetArtifactTypesByIDResponse struct { +type GetExecutionsByTypeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The result is not index-aligned: if an id is not found, it is not - // returned. - ArtifactTypes []*ArtifactType `protobuf:"bytes,1,rep,name=artifact_types,json=artifactTypes" json:"artifact_types,omitempty"` + Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` + // Token to use to retrieve next page of results if list options are used in + // the request. + NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` } -func (x *GetArtifactTypesByIDResponse) Reset() { - *x = GetArtifactTypesByIDResponse{} +func (x *GetExecutionsByTypeResponse) Reset() { + *x = GetExecutionsByTypeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[59] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetArtifactTypesByIDResponse) String() string { +func (x *GetExecutionsByTypeResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetArtifactTypesByIDResponse) ProtoMessage() {} +func (*GetExecutionsByTypeResponse) ProtoMessage() {} -func (x *GetArtifactTypesByIDResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[59] +func (x *GetExecutionsByTypeResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3496,45 +3929,56 @@ func (x *GetArtifactTypesByIDResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetArtifactTypesByIDResponse.ProtoReflect.Descriptor instead. -func (*GetArtifactTypesByIDResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{59} +// Deprecated: Use GetExecutionsByTypeResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionsByTypeResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{61} } -func (x *GetArtifactTypesByIDResponse) GetArtifactTypes() []*ArtifactType { +func (x *GetExecutionsByTypeResponse) GetExecutions() []*Execution { if x != nil { - return x.ArtifactTypes + return x.Executions } return nil } -type GetExecutionTypesByIDRequest struct { +func (x *GetExecutionsByTypeResponse) GetNextPageToken() string { + if x != nil && x.NextPageToken != nil { + return *x.NextPageToken + } + return "" +} + +type GetExecutionByTypeAndNameRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeIds []int64 `protobuf:"varint,1,rep,name=type_ids,json=typeIds" json:"type_ids,omitempty"` + TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` + // If not set, it looks for the type with type_name and execution_name with + // default type_version. + TypeVersion *string `protobuf:"bytes,3,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` + ExecutionName *string `protobuf:"bytes,2,opt,name=execution_name,json=executionName" json:"execution_name,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetExecutionTypesByIDRequest) Reset() { - *x = GetExecutionTypesByIDRequest{} +func (x *GetExecutionByTypeAndNameRequest) Reset() { + *x = GetExecutionByTypeAndNameRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[60] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionTypesByIDRequest) String() string { +func (x *GetExecutionByTypeAndNameRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionTypesByIDRequest) ProtoMessage() {} +func (*GetExecutionByTypeAndNameRequest) ProtoMessage() {} -func (x *GetExecutionTypesByIDRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[60] +func (x *GetExecutionByTypeAndNameRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3545,52 +3989,64 @@ func (x *GetExecutionTypesByIDRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionTypesByIDRequest.ProtoReflect.Descriptor instead. -func (*GetExecutionTypesByIDRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{60} +// Deprecated: Use GetExecutionByTypeAndNameRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionByTypeAndNameRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{62} } -func (x *GetExecutionTypesByIDRequest) GetTypeIds() []int64 { - if x != nil { - return x.TypeIds +func (x *GetExecutionByTypeAndNameRequest) GetTypeName() string { + if x != nil && x.TypeName != nil { + return *x.TypeName } - return nil + return "" } -func (x *GetExecutionTypesByIDRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetExecutionByTypeAndNameRequest) GetTypeVersion() string { + if x != nil && x.TypeVersion != nil { + return *x.TypeVersion + } + return "" +} + +func (x *GetExecutionByTypeAndNameRequest) GetExecutionName() string { + if x != nil && x.ExecutionName != nil { + return *x.ExecutionName + } + return "" +} + +func (x *GetExecutionByTypeAndNameRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetExecutionTypesByIDResponse struct { +type GetExecutionByTypeAndNameResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The result is not index-aligned: if an id is not found, it is not - // returned. - ExecutionTypes []*ExecutionType `protobuf:"bytes,1,rep,name=execution_types,json=executionTypes" json:"execution_types,omitempty"` + Execution *Execution `protobuf:"bytes,1,opt,name=execution" json:"execution,omitempty"` } -func (x *GetExecutionTypesByIDResponse) Reset() { - *x = GetExecutionTypesByIDResponse{} +func (x *GetExecutionByTypeAndNameResponse) Reset() { + *x = GetExecutionByTypeAndNameResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[61] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionTypesByIDResponse) String() string { +func (x *GetExecutionByTypeAndNameResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionTypesByIDResponse) ProtoMessage() {} +func (*GetExecutionByTypeAndNameResponse) ProtoMessage() {} -func (x *GetExecutionTypesByIDResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[61] +func (x *GetExecutionByTypeAndNameResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3601,47 +4057,46 @@ func (x *GetExecutionTypesByIDResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionTypesByIDResponse.ProtoReflect.Descriptor instead. -func (*GetExecutionTypesByIDResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{61} +// Deprecated: Use GetExecutionByTypeAndNameResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionByTypeAndNameResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{63} } -func (x *GetExecutionTypesByIDResponse) GetExecutionTypes() []*ExecutionType { +func (x *GetExecutionByTypeAndNameResponse) GetExecution() *Execution { if x != nil { - return x.ExecutionTypes + return x.Execution } return nil } -type GetContextTypeRequest struct { +type GetExecutionsByIDRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` - // If not set, it looks for the type with type_name with default type_version. - TypeVersion *string `protobuf:"bytes,2,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` + // A list of execution ids to retrieve. + ExecutionIds []int64 `protobuf:"varint,1,rep,name=execution_ids,json=executionIds" json:"execution_ids,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextTypeRequest) Reset() { - *x = GetContextTypeRequest{} +func (x *GetExecutionsByIDRequest) Reset() { + *x = GetExecutionsByIDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[62] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextTypeRequest) String() string { +func (x *GetExecutionsByIDRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextTypeRequest) ProtoMessage() {} +func (*GetExecutionsByIDRequest) ProtoMessage() {} -func (x *GetContextTypeRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[62] +func (x *GetExecutionsByIDRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3652,58 +4107,52 @@ func (x *GetContextTypeRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextTypeRequest.ProtoReflect.Descriptor instead. -func (*GetContextTypeRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{62} -} - -func (x *GetContextTypeRequest) GetTypeName() string { - if x != nil && x.TypeName != nil { - return *x.TypeName - } - return "" +// Deprecated: Use GetExecutionsByIDRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionsByIDRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{64} } -func (x *GetContextTypeRequest) GetTypeVersion() string { - if x != nil && x.TypeVersion != nil { - return *x.TypeVersion +func (x *GetExecutionsByIDRequest) GetExecutionIds() []int64 { + if x != nil { + return x.ExecutionIds } - return "" + return nil } -func (x *GetContextTypeRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetExecutionsByIDRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextTypeResponse struct { +type GetExecutionsByIDResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Gets a context type, or clear if it does not exist. - ContextType *ContextType `protobuf:"bytes,1,opt,name=context_type,json=contextType" json:"context_type,omitempty"` + // The result is not index-aligned: if an id is not found, it is not + // returned. + Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` } -func (x *GetContextTypeResponse) Reset() { - *x = GetContextTypeResponse{} +func (x *GetExecutionsByIDResponse) Reset() { + *x = GetExecutionsByIDResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[63] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextTypeResponse) String() string { +func (x *GetExecutionsByIDResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextTypeResponse) ProtoMessage() {} +func (*GetExecutionsByIDResponse) ProtoMessage() {} -func (x *GetContextTypeResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[63] +func (x *GetExecutionsByIDResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3714,45 +4163,47 @@ func (x *GetContextTypeResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextTypeResponse.ProtoReflect.Descriptor instead. -func (*GetContextTypeResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{63} +// Deprecated: Use GetExecutionsByIDResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionsByIDResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{65} } -func (x *GetContextTypeResponse) GetContextType() *ContextType { +func (x *GetExecutionsByIDResponse) GetExecutions() []*Execution { if x != nil { - return x.ContextType + return x.Executions } return nil } -type GetContextTypesByIDRequest struct { +type GetExecutionTypeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeIds []int64 `protobuf:"varint,1,rep,name=type_ids,json=typeIds" json:"type_ids,omitempty"` + TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` + // If not set, it looks for the type with type_name with default type_version. + TypeVersion *string `protobuf:"bytes,2,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextTypesByIDRequest) Reset() { - *x = GetContextTypesByIDRequest{} +func (x *GetExecutionTypeRequest) Reset() { + *x = GetExecutionTypeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[64] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextTypesByIDRequest) String() string { +func (x *GetExecutionTypeRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextTypesByIDRequest) ProtoMessage() {} +func (*GetExecutionTypeRequest) ProtoMessage() {} -func (x *GetContextTypesByIDRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[64] +func (x *GetExecutionTypeRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[66] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3763,52 +4214,58 @@ func (x *GetContextTypesByIDRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextTypesByIDRequest.ProtoReflect.Descriptor instead. -func (*GetContextTypesByIDRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{64} +// Deprecated: Use GetExecutionTypeRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionTypeRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{66} } -func (x *GetContextTypesByIDRequest) GetTypeIds() []int64 { - if x != nil { - return x.TypeIds +func (x *GetExecutionTypeRequest) GetTypeName() string { + if x != nil && x.TypeName != nil { + return *x.TypeName } - return nil + return "" } -func (x *GetContextTypesByIDRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetExecutionTypeRequest) GetTypeVersion() string { + if x != nil && x.TypeVersion != nil { + return *x.TypeVersion + } + return "" +} + +func (x *GetExecutionTypeRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextTypesByIDResponse struct { +type GetExecutionTypeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The result is not index-aligned: if an id is not found, it is not - // returned. - ContextTypes []*ContextType `protobuf:"bytes,1,rep,name=context_types,json=contextTypes" json:"context_types,omitempty"` + // Gets an execution type, or clear if it does not exist. + ExecutionType *ExecutionType `protobuf:"bytes,1,opt,name=execution_type,json=executionType" json:"execution_type,omitempty"` } -func (x *GetContextTypesByIDResponse) Reset() { - *x = GetContextTypesByIDResponse{} +func (x *GetExecutionTypeResponse) Reset() { + *x = GetExecutionTypeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[65] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextTypesByIDResponse) String() string { +func (x *GetExecutionTypeResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextTypesByIDResponse) ProtoMessage() {} +func (*GetExecutionTypeResponse) ProtoMessage() {} -func (x *GetContextTypesByIDResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[65] +func (x *GetExecutionTypeResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[67] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3819,51 +4276,46 @@ func (x *GetContextTypesByIDResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextTypesByIDResponse.ProtoReflect.Descriptor instead. -func (*GetContextTypesByIDResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{65} +// Deprecated: Use GetExecutionTypeResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionTypeResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{67} } -func (x *GetContextTypesByIDResponse) GetContextTypes() []*ContextType { +func (x *GetExecutionTypeResponse) GetExecutionType() *ExecutionType { if x != nil { - return x.ContextTypes + return x.ExecutionType } return nil } -// Request to retrieve Contexts using List options. -// If option is not specified then all Contexts are returned. -type GetContextsRequest struct { +// Gets all events with matching execution ids. +type GetEventsByExecutionIDsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Specify options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. - Options *ListOperationOptions `protobuf:"bytes,1,opt,name=options" json:"options,omitempty"` + ExecutionIds []int64 `protobuf:"varint,1,rep,name=execution_ids,json=executionIds" json:"execution_ids,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextsRequest) Reset() { - *x = GetContextsRequest{} +func (x *GetEventsByExecutionIDsRequest) Reset() { + *x = GetEventsByExecutionIDsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[66] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsRequest) String() string { +func (x *GetEventsByExecutionIDsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsRequest) ProtoMessage() {} +func (*GetEventsByExecutionIDsRequest) ProtoMessage() {} -func (x *GetContextsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[66] +func (x *GetEventsByExecutionIDsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3874,54 +4326,50 @@ func (x *GetContextsRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsRequest.ProtoReflect.Descriptor instead. -func (*GetContextsRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{66} +// Deprecated: Use GetEventsByExecutionIDsRequest.ProtoReflect.Descriptor instead. +func (*GetEventsByExecutionIDsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{68} } -func (x *GetContextsRequest) GetOptions() *ListOperationOptions { +func (x *GetEventsByExecutionIDsRequest) GetExecutionIds() []int64 { if x != nil { - return x.Options + return x.ExecutionIds } return nil } -func (x *GetContextsRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetEventsByExecutionIDsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextsResponse struct { +type GetEventsByExecutionIDsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Returned contexts. - Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` - // Token to use to retrieve next page of results if list options are used in - // the request. - NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` + Events []*Event `protobuf:"bytes,1,rep,name=events" json:"events,omitempty"` } -func (x *GetContextsResponse) Reset() { - *x = GetContextsResponse{} +func (x *GetEventsByExecutionIDsResponse) Reset() { + *x = GetEventsByExecutionIDsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[67] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsResponse) String() string { +func (x *GetEventsByExecutionIDsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsResponse) ProtoMessage() {} +func (*GetEventsByExecutionIDsResponse) ProtoMessage() {} -func (x *GetContextsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[67] +func (x *GetEventsByExecutionIDsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3932,60 +4380,45 @@ func (x *GetContextsResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsResponse.ProtoReflect.Descriptor instead. -func (*GetContextsResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{67} +// Deprecated: Use GetEventsByExecutionIDsResponse.ProtoReflect.Descriptor instead. +func (*GetEventsByExecutionIDsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{69} } -func (x *GetContextsResponse) GetContexts() []*Context { +func (x *GetEventsByExecutionIDsResponse) GetEvents() []*Event { if x != nil { - return x.Contexts + return x.Events } return nil } -func (x *GetContextsResponse) GetNextPageToken() string { - if x != nil && x.NextPageToken != nil { - return *x.NextPageToken - } - return "" -} - -type GetContextsByTypeRequest struct { +type GetEventsByArtifactIDsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` - // Specify options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. - Options *ListOperationOptions `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` - // If not set, it looks for the type with type_name and options with default - // type_version. - TypeVersion *string `protobuf:"bytes,3,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` + ArtifactIds []int64 `protobuf:"varint,1,rep,name=artifact_ids,json=artifactIds" json:"artifact_ids,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextsByTypeRequest) Reset() { - *x = GetContextsByTypeRequest{} +func (x *GetEventsByArtifactIDsRequest) Reset() { + *x = GetEventsByArtifactIDsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[68] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByTypeRequest) String() string { +func (x *GetEventsByArtifactIDsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByTypeRequest) ProtoMessage() {} +func (*GetEventsByArtifactIDsRequest) ProtoMessage() {} -func (x *GetContextsByTypeRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[68] +func (x *GetEventsByArtifactIDsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[70] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3996,67 +4429,50 @@ func (x *GetContextsByTypeRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByTypeRequest.ProtoReflect.Descriptor instead. -func (*GetContextsByTypeRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{68} -} - -func (x *GetContextsByTypeRequest) GetTypeName() string { - if x != nil && x.TypeName != nil { - return *x.TypeName - } - return "" +// Deprecated: Use GetEventsByArtifactIDsRequest.ProtoReflect.Descriptor instead. +func (*GetEventsByArtifactIDsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{70} } -func (x *GetContextsByTypeRequest) GetOptions() *ListOperationOptions { +func (x *GetEventsByArtifactIDsRequest) GetArtifactIds() []int64 { if x != nil { - return x.Options + return x.ArtifactIds } return nil } -func (x *GetContextsByTypeRequest) GetTypeVersion() string { - if x != nil && x.TypeVersion != nil { - return *x.TypeVersion - } - return "" -} - -func (x *GetContextsByTypeRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetEventsByArtifactIDsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextsByTypeResponse struct { +type GetEventsByArtifactIDsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` - // Token to use to retrieve next page of results if list options are used in - // the request. - NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` + Events []*Event `protobuf:"bytes,1,rep,name=events" json:"events,omitempty"` } -func (x *GetContextsByTypeResponse) Reset() { - *x = GetContextsByTypeResponse{} +func (x *GetEventsByArtifactIDsResponse) Reset() { + *x = GetEventsByArtifactIDsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[69] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByTypeResponse) String() string { +func (x *GetEventsByArtifactIDsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByTypeResponse) ProtoMessage() {} +func (*GetEventsByArtifactIDsResponse) ProtoMessage() {} -func (x *GetContextsByTypeResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[69] +func (x *GetEventsByArtifactIDsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[71] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4067,56 +4483,45 @@ func (x *GetContextsByTypeResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByTypeResponse.ProtoReflect.Descriptor instead. -func (*GetContextsByTypeResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{69} +// Deprecated: Use GetEventsByArtifactIDsResponse.ProtoReflect.Descriptor instead. +func (*GetEventsByArtifactIDsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{71} } -func (x *GetContextsByTypeResponse) GetContexts() []*Context { +func (x *GetEventsByArtifactIDsResponse) GetEvents() []*Event { if x != nil { - return x.Contexts + return x.Events } return nil } -func (x *GetContextsByTypeResponse) GetNextPageToken() string { - if x != nil && x.NextPageToken != nil { - return *x.NextPageToken - } - return "" -} - -type GetContextByTypeAndNameRequest struct { +type GetArtifactTypesByIDRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` - // If not set, it looks for the type with type_name and context_name with - // default type_version. - TypeVersion *string `protobuf:"bytes,3,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` - ContextName *string `protobuf:"bytes,2,opt,name=context_name,json=contextName" json:"context_name,omitempty"` + TypeIds []int64 `protobuf:"varint,1,rep,name=type_ids,json=typeIds" json:"type_ids,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextByTypeAndNameRequest) Reset() { - *x = GetContextByTypeAndNameRequest{} +func (x *GetArtifactTypesByIDRequest) Reset() { + *x = GetArtifactTypesByIDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[70] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextByTypeAndNameRequest) String() string { +func (x *GetArtifactTypesByIDRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextByTypeAndNameRequest) ProtoMessage() {} +func (*GetArtifactTypesByIDRequest) ProtoMessage() {} -func (x *GetContextByTypeAndNameRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[70] +func (x *GetArtifactTypesByIDRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4127,64 +4532,52 @@ func (x *GetContextByTypeAndNameRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextByTypeAndNameRequest.ProtoReflect.Descriptor instead. -func (*GetContextByTypeAndNameRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{70} -} - -func (x *GetContextByTypeAndNameRequest) GetTypeName() string { - if x != nil && x.TypeName != nil { - return *x.TypeName - } - return "" -} - -func (x *GetContextByTypeAndNameRequest) GetTypeVersion() string { - if x != nil && x.TypeVersion != nil { - return *x.TypeVersion - } - return "" +// Deprecated: Use GetArtifactTypesByIDRequest.ProtoReflect.Descriptor instead. +func (*GetArtifactTypesByIDRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{72} } -func (x *GetContextByTypeAndNameRequest) GetContextName() string { - if x != nil && x.ContextName != nil { - return *x.ContextName +func (x *GetArtifactTypesByIDRequest) GetTypeIds() []int64 { + if x != nil { + return x.TypeIds } - return "" + return nil } -func (x *GetContextByTypeAndNameRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetArtifactTypesByIDRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextByTypeAndNameResponse struct { +type GetArtifactTypesByIDResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Context *Context `protobuf:"bytes,1,opt,name=context" json:"context,omitempty"` + // The result is not index-aligned: if an id is not found, it is not + // returned. + ArtifactTypes []*ArtifactType `protobuf:"bytes,1,rep,name=artifact_types,json=artifactTypes" json:"artifact_types,omitempty"` } -func (x *GetContextByTypeAndNameResponse) Reset() { - *x = GetContextByTypeAndNameResponse{} +func (x *GetArtifactTypesByIDResponse) Reset() { + *x = GetArtifactTypesByIDResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[71] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextByTypeAndNameResponse) String() string { +func (x *GetArtifactTypesByIDResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextByTypeAndNameResponse) ProtoMessage() {} +func (*GetArtifactTypesByIDResponse) ProtoMessage() {} -func (x *GetContextByTypeAndNameResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[71] +func (x *GetArtifactTypesByIDResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[73] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4195,46 +4588,45 @@ func (x *GetContextByTypeAndNameResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextByTypeAndNameResponse.ProtoReflect.Descriptor instead. -func (*GetContextByTypeAndNameResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{71} +// Deprecated: Use GetArtifactTypesByIDResponse.ProtoReflect.Descriptor instead. +func (*GetArtifactTypesByIDResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{73} } -func (x *GetContextByTypeAndNameResponse) GetContext() *Context { +func (x *GetArtifactTypesByIDResponse) GetArtifactTypes() []*ArtifactType { if x != nil { - return x.Context + return x.ArtifactTypes } return nil } -type GetContextsByIDRequest struct { +type GetExecutionTypesByIDRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // A list of context ids to retrieve. - ContextIds []int64 `protobuf:"varint,1,rep,name=context_ids,json=contextIds" json:"context_ids,omitempty"` + TypeIds []int64 `protobuf:"varint,1,rep,name=type_ids,json=typeIds" json:"type_ids,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextsByIDRequest) Reset() { - *x = GetContextsByIDRequest{} +func (x *GetExecutionTypesByIDRequest) Reset() { + *x = GetExecutionTypesByIDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[72] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByIDRequest) String() string { +func (x *GetExecutionTypesByIDRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByIDRequest) ProtoMessage() {} +func (*GetExecutionTypesByIDRequest) ProtoMessage() {} -func (x *GetContextsByIDRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[72] +func (x *GetExecutionTypesByIDRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4245,52 +4637,52 @@ func (x *GetContextsByIDRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByIDRequest.ProtoReflect.Descriptor instead. -func (*GetContextsByIDRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{72} +// Deprecated: Use GetExecutionTypesByIDRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionTypesByIDRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{74} } -func (x *GetContextsByIDRequest) GetContextIds() []int64 { +func (x *GetExecutionTypesByIDRequest) GetTypeIds() []int64 { if x != nil { - return x.ContextIds + return x.TypeIds } return nil } -func (x *GetContextsByIDRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetExecutionTypesByIDRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextsByIDResponse struct { +type GetExecutionTypesByIDResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields // The result is not index-aligned: if an id is not found, it is not // returned. - Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` + ExecutionTypes []*ExecutionType `protobuf:"bytes,1,rep,name=execution_types,json=executionTypes" json:"execution_types,omitempty"` } -func (x *GetContextsByIDResponse) Reset() { - *x = GetContextsByIDResponse{} +func (x *GetExecutionTypesByIDResponse) Reset() { + *x = GetExecutionTypesByIDResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[73] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByIDResponse) String() string { +func (x *GetExecutionTypesByIDResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByIDResponse) ProtoMessage() {} +func (*GetExecutionTypesByIDResponse) ProtoMessage() {} -func (x *GetContextsByIDResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[73] +func (x *GetExecutionTypesByIDResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4301,45 +4693,47 @@ func (x *GetContextsByIDResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByIDResponse.ProtoReflect.Descriptor instead. -func (*GetContextsByIDResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{73} +// Deprecated: Use GetExecutionTypesByIDResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionTypesByIDResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{75} } -func (x *GetContextsByIDResponse) GetContexts() []*Context { +func (x *GetExecutionTypesByIDResponse) GetExecutionTypes() []*ExecutionType { if x != nil { - return x.Contexts + return x.ExecutionTypes } return nil } -type GetContextsByArtifactRequest struct { +type GetContextTypeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ArtifactId *int64 `protobuf:"varint,1,opt,name=artifact_id,json=artifactId" json:"artifact_id,omitempty"` + TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` + // If not set, it looks for the type with type_name with default type_version. + TypeVersion *string `protobuf:"bytes,2,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextsByArtifactRequest) Reset() { - *x = GetContextsByArtifactRequest{} +func (x *GetContextTypeRequest) Reset() { + *x = GetContextTypeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[74] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByArtifactRequest) String() string { +func (x *GetContextTypeRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByArtifactRequest) ProtoMessage() {} +func (*GetContextTypeRequest) ProtoMessage() {} -func (x *GetContextsByArtifactRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[74] +func (x *GetContextTypeRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[76] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4350,50 +4744,58 @@ func (x *GetContextsByArtifactRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByArtifactRequest.ProtoReflect.Descriptor instead. -func (*GetContextsByArtifactRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{74} +// Deprecated: Use GetContextTypeRequest.ProtoReflect.Descriptor instead. +func (*GetContextTypeRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{76} } -func (x *GetContextsByArtifactRequest) GetArtifactId() int64 { - if x != nil && x.ArtifactId != nil { - return *x.ArtifactId +func (x *GetContextTypeRequest) GetTypeName() string { + if x != nil && x.TypeName != nil { + return *x.TypeName } - return 0 + return "" } -func (x *GetContextsByArtifactRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextTypeRequest) GetTypeVersion() string { + if x != nil && x.TypeVersion != nil { + return *x.TypeVersion + } + return "" +} + +func (x *GetContextTypeRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextsByArtifactResponse struct { +type GetContextTypeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` + // Gets a context type, or clear if it does not exist. + ContextType *ContextType `protobuf:"bytes,1,opt,name=context_type,json=contextType" json:"context_type,omitempty"` } -func (x *GetContextsByArtifactResponse) Reset() { - *x = GetContextsByArtifactResponse{} +func (x *GetContextTypeResponse) Reset() { + *x = GetContextTypeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[75] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByArtifactResponse) String() string { +func (x *GetContextTypeResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByArtifactResponse) ProtoMessage() {} +func (*GetContextTypeResponse) ProtoMessage() {} -func (x *GetContextsByArtifactResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[75] +func (x *GetContextTypeResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[77] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4404,45 +4806,45 @@ func (x *GetContextsByArtifactResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByArtifactResponse.ProtoReflect.Descriptor instead. -func (*GetContextsByArtifactResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{75} +// Deprecated: Use GetContextTypeResponse.ProtoReflect.Descriptor instead. +func (*GetContextTypeResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{77} } -func (x *GetContextsByArtifactResponse) GetContexts() []*Context { +func (x *GetContextTypeResponse) GetContextType() *ContextType { if x != nil { - return x.Contexts + return x.ContextType } return nil } -type GetContextsByExecutionRequest struct { +type GetContextTypesByIDRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ExecutionId *int64 `protobuf:"varint,1,opt,name=execution_id,json=executionId" json:"execution_id,omitempty"` + TypeIds []int64 `protobuf:"varint,1,rep,name=type_ids,json=typeIds" json:"type_ids,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetContextsByExecutionRequest) Reset() { - *x = GetContextsByExecutionRequest{} +func (x *GetContextTypesByIDRequest) Reset() { + *x = GetContextTypesByIDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[76] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByExecutionRequest) String() string { +func (x *GetContextTypesByIDRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByExecutionRequest) ProtoMessage() {} +func (*GetContextTypesByIDRequest) ProtoMessage() {} -func (x *GetContextsByExecutionRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[76] +func (x *GetContextTypesByIDRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[78] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4453,50 +4855,52 @@ func (x *GetContextsByExecutionRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByExecutionRequest.ProtoReflect.Descriptor instead. -func (*GetContextsByExecutionRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{76} +// Deprecated: Use GetContextTypesByIDRequest.ProtoReflect.Descriptor instead. +func (*GetContextTypesByIDRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{78} } -func (x *GetContextsByExecutionRequest) GetExecutionId() int64 { - if x != nil && x.ExecutionId != nil { - return *x.ExecutionId +func (x *GetContextTypesByIDRequest) GetTypeIds() []int64 { + if x != nil { + return x.TypeIds } - return 0 + return nil } -func (x *GetContextsByExecutionRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextTypesByIDRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetContextsByExecutionResponse struct { +type GetContextTypesByIDResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` + // The result is not index-aligned: if an id is not found, it is not + // returned. + ContextTypes []*ContextType `protobuf:"bytes,1,rep,name=context_types,json=contextTypes" json:"context_types,omitempty"` } -func (x *GetContextsByExecutionResponse) Reset() { - *x = GetContextsByExecutionResponse{} +func (x *GetContextTypesByIDResponse) Reset() { + *x = GetContextTypesByIDResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[77] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetContextsByExecutionResponse) String() string { +func (x *GetContextTypesByIDResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetContextsByExecutionResponse) ProtoMessage() {} +func (*GetContextTypesByIDResponse) ProtoMessage() {} -func (x *GetContextsByExecutionResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[77] +func (x *GetContextTypesByIDResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[79] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4507,45 +4911,50 @@ func (x *GetContextsByExecutionResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetContextsByExecutionResponse.ProtoReflect.Descriptor instead. -func (*GetContextsByExecutionResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{77} +// Deprecated: Use GetContextTypesByIDResponse.ProtoReflect.Descriptor instead. +func (*GetContextTypesByIDResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{79} } -func (x *GetContextsByExecutionResponse) GetContexts() []*Context { +func (x *GetContextTypesByIDResponse) GetContextTypes() []*ContextType { if x != nil { - return x.Contexts + return x.ContextTypes } return nil } -type GetParentContextsByContextRequest struct { +// Request to retrieve Contexts using List options. +// If option is not specified then all Contexts are returned. +type GetContextsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` + // Specify options. + // Please refer to the documentation of ListOperationOptions for the supported + // functionalities. + Options *ListOperationOptions `protobuf:"bytes,1,opt,name=options" json:"options,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetParentContextsByContextRequest) Reset() { - *x = GetParentContextsByContextRequest{} +func (x *GetContextsRequest) Reset() { + *x = GetContextsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[78] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetParentContextsByContextRequest) String() string { +func (x *GetContextsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetParentContextsByContextRequest) ProtoMessage() {} +func (*GetContextsRequest) ProtoMessage() {} -func (x *GetParentContextsByContextRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[78] +func (x *GetContextsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[80] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4556,50 +4965,54 @@ func (x *GetParentContextsByContextRequest) ProtoReflect() protoreflect.Message return mi.MessageOf(x) } -// Deprecated: Use GetParentContextsByContextRequest.ProtoReflect.Descriptor instead. -func (*GetParentContextsByContextRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{78} +// Deprecated: Use GetContextsRequest.ProtoReflect.Descriptor instead. +func (*GetContextsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{80} } -func (x *GetParentContextsByContextRequest) GetContextId() int64 { - if x != nil && x.ContextId != nil { - return *x.ContextId +func (x *GetContextsRequest) GetOptions() *ListOperationOptions { + if x != nil { + return x.Options } - return 0 + return nil } -func (x *GetParentContextsByContextRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextsRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetParentContextsByContextResponse struct { +type GetContextsResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // Returned contexts. Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` + // Token to use to retrieve next page of results if list options are used in + // the request. + NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` } -func (x *GetParentContextsByContextResponse) Reset() { - *x = GetParentContextsByContextResponse{} +func (x *GetContextsResponse) Reset() { + *x = GetContextsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[79] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetParentContextsByContextResponse) String() string { +func (x *GetContextsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetParentContextsByContextResponse) ProtoMessage() {} +func (*GetContextsResponse) ProtoMessage() {} -func (x *GetParentContextsByContextResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[79] +func (x *GetContextsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[81] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4610,45 +5023,66 @@ func (x *GetParentContextsByContextResponse) ProtoReflect() protoreflect.Message return mi.MessageOf(x) } -// Deprecated: Use GetParentContextsByContextResponse.ProtoReflect.Descriptor instead. -func (*GetParentContextsByContextResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{79} +// Deprecated: Use GetContextsResponse.ProtoReflect.Descriptor instead. +func (*GetContextsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{81} } -func (x *GetParentContextsByContextResponse) GetContexts() []*Context { +func (x *GetContextsResponse) GetContexts() []*Context { if x != nil { return x.Contexts } return nil } -type GetChildrenContextsByContextRequest struct { +func (x *GetContextsResponse) GetNextPageToken() string { + if x != nil && x.NextPageToken != nil { + return *x.NextPageToken + } + return "" +} + +type GetContextsByTypeRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` + TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` + // Specify options. + // Currently supports: + // 1. Field to order the results. + // 2. Page size. + // If set, the request will + // first fetch all contexts with specified `type_name` and `type_version`, + // then order by a specifield field + // finally find the correct page and return #Contexts of the page size. + // Higher-level APIs may only use the functionalies partially. + // Please reference the API documentation for the API behaviors. + Options *ListOperationOptions `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` + // If not set, it looks for the type with type_name and options with default + // type_version. + TypeVersion *string `protobuf:"bytes,3,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetChildrenContextsByContextRequest) Reset() { - *x = GetChildrenContextsByContextRequest{} +func (x *GetContextsByTypeRequest) Reset() { + *x = GetContextsByTypeRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[80] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetChildrenContextsByContextRequest) String() string { +func (x *GetContextsByTypeRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetChildrenContextsByContextRequest) ProtoMessage() {} +func (*GetContextsByTypeRequest) ProtoMessage() {} -func (x *GetChildrenContextsByContextRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[80] +func (x *GetContextsByTypeRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[82] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4659,50 +5093,67 @@ func (x *GetChildrenContextsByContextRequest) ProtoReflect() protoreflect.Messag return mi.MessageOf(x) } -// Deprecated: Use GetChildrenContextsByContextRequest.ProtoReflect.Descriptor instead. -func (*GetChildrenContextsByContextRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{80} +// Deprecated: Use GetContextsByTypeRequest.ProtoReflect.Descriptor instead. +func (*GetContextsByTypeRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{82} } -func (x *GetChildrenContextsByContextRequest) GetContextId() int64 { - if x != nil && x.ContextId != nil { - return *x.ContextId +func (x *GetContextsByTypeRequest) GetTypeName() string { + if x != nil && x.TypeName != nil { + return *x.TypeName } - return 0 + return "" } -func (x *GetChildrenContextsByContextRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextsByTypeRequest) GetOptions() *ListOperationOptions { + if x != nil { + return x.Options + } + return nil +} + +func (x *GetContextsByTypeRequest) GetTypeVersion() string { + if x != nil && x.TypeVersion != nil { + return *x.TypeVersion + } + return "" +} + +func (x *GetContextsByTypeRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetChildrenContextsByContextResponse struct { +type GetContextsByTypeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` + // Token to use to retrieve next page of results if list options are used in + // the request. + NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` } -func (x *GetChildrenContextsByContextResponse) Reset() { - *x = GetChildrenContextsByContextResponse{} +func (x *GetContextsByTypeResponse) Reset() { + *x = GetContextsByTypeResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[81] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetChildrenContextsByContextResponse) String() string { +func (x *GetContextsByTypeResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetChildrenContextsByContextResponse) ProtoMessage() {} +func (*GetContextsByTypeResponse) ProtoMessage() {} -func (x *GetChildrenContextsByContextResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[81] +func (x *GetContextsByTypeResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[83] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4713,50 +5164,56 @@ func (x *GetChildrenContextsByContextResponse) ProtoReflect() protoreflect.Messa return mi.MessageOf(x) } -// Deprecated: Use GetChildrenContextsByContextResponse.ProtoReflect.Descriptor instead. -func (*GetChildrenContextsByContextResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{81} +// Deprecated: Use GetContextsByTypeResponse.ProtoReflect.Descriptor instead. +func (*GetContextsByTypeResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{83} } -func (x *GetChildrenContextsByContextResponse) GetContexts() []*Context { +func (x *GetContextsByTypeResponse) GetContexts() []*Context { if x != nil { return x.Contexts } return nil } -type GetArtifactsByContextRequest struct { +func (x *GetContextsByTypeResponse) GetNextPageToken() string { + if x != nil && x.NextPageToken != nil { + return *x.NextPageToken + } + return "" +} + +type GetContextByTypeAndNameRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` - // Specify List options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. - Options *ListOperationOptions `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` + TypeName *string `protobuf:"bytes,1,opt,name=type_name,json=typeName" json:"type_name,omitempty"` + // If not set, it looks for the type with type_name and context_name with + // default type_version. + TypeVersion *string `protobuf:"bytes,3,opt,name=type_version,json=typeVersion" json:"type_version,omitempty"` + ContextName *string `protobuf:"bytes,2,opt,name=context_name,json=contextName" json:"context_name,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,4,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetArtifactsByContextRequest) Reset() { - *x = GetArtifactsByContextRequest{} +func (x *GetContextByTypeAndNameRequest) Reset() { + *x = GetContextByTypeAndNameRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[82] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetArtifactsByContextRequest) String() string { +func (x *GetContextByTypeAndNameRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetArtifactsByContextRequest) ProtoMessage() {} +func (*GetContextByTypeAndNameRequest) ProtoMessage() {} -func (x *GetArtifactsByContextRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[82] +func (x *GetContextByTypeAndNameRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[84] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4767,60 +5224,64 @@ func (x *GetArtifactsByContextRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetArtifactsByContextRequest.ProtoReflect.Descriptor instead. -func (*GetArtifactsByContextRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{82} +// Deprecated: Use GetContextByTypeAndNameRequest.ProtoReflect.Descriptor instead. +func (*GetContextByTypeAndNameRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{84} } -func (x *GetArtifactsByContextRequest) GetContextId() int64 { - if x != nil && x.ContextId != nil { - return *x.ContextId +func (x *GetContextByTypeAndNameRequest) GetTypeName() string { + if x != nil && x.TypeName != nil { + return *x.TypeName } - return 0 + return "" } -func (x *GetArtifactsByContextRequest) GetOptions() *ListOperationOptions { - if x != nil { - return x.Options +func (x *GetContextByTypeAndNameRequest) GetTypeVersion() string { + if x != nil && x.TypeVersion != nil { + return *x.TypeVersion } - return nil + return "" } -func (x *GetArtifactsByContextRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextByTypeAndNameRequest) GetContextName() string { + if x != nil && x.ContextName != nil { + return *x.ContextName + } + return "" +} + +func (x *GetContextByTypeAndNameRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetArtifactsByContextResponse struct { +type GetContextByTypeAndNameResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Artifacts []*Artifact `protobuf:"bytes,1,rep,name=artifacts" json:"artifacts,omitempty"` - // Token to use to retrieve next page of results if list options are used in - // the request. - NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` + Context *Context `protobuf:"bytes,1,opt,name=context" json:"context,omitempty"` } -func (x *GetArtifactsByContextResponse) Reset() { - *x = GetArtifactsByContextResponse{} +func (x *GetContextByTypeAndNameResponse) Reset() { + *x = GetContextByTypeAndNameResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[83] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetArtifactsByContextResponse) String() string { +func (x *GetContextByTypeAndNameResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetArtifactsByContextResponse) ProtoMessage() {} +func (*GetContextByTypeAndNameResponse) ProtoMessage() {} -func (x *GetArtifactsByContextResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[83] +func (x *GetContextByTypeAndNameResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[85] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4831,57 +5292,46 @@ func (x *GetArtifactsByContextResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetArtifactsByContextResponse.ProtoReflect.Descriptor instead. -func (*GetArtifactsByContextResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{83} +// Deprecated: Use GetContextByTypeAndNameResponse.ProtoReflect.Descriptor instead. +func (*GetContextByTypeAndNameResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{85} } -func (x *GetArtifactsByContextResponse) GetArtifacts() []*Artifact { +func (x *GetContextByTypeAndNameResponse) GetContext() *Context { if x != nil { - return x.Artifacts + return x.Context } return nil } -func (x *GetArtifactsByContextResponse) GetNextPageToken() string { - if x != nil && x.NextPageToken != nil { - return *x.NextPageToken - } - return "" -} - -type GetExecutionsByContextRequest struct { +type GetContextsByIDRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` - // Specify List options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. - Options *ListOperationOptions `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` + // A list of context ids to retrieve. + ContextIds []int64 `protobuf:"varint,1,rep,name=context_ids,json=contextIds" json:"context_ids,omitempty"` // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetExecutionsByContextRequest) Reset() { - *x = GetExecutionsByContextRequest{} +func (x *GetContextsByIDRequest) Reset() { + *x = GetContextsByIDRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[84] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionsByContextRequest) String() string { +func (x *GetContextsByIDRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionsByContextRequest) ProtoMessage() {} +func (*GetContextsByIDRequest) ProtoMessage() {} -func (x *GetExecutionsByContextRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[84] +func (x *GetContextsByIDRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[86] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4892,62 +5342,52 @@ func (x *GetExecutionsByContextRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionsByContextRequest.ProtoReflect.Descriptor instead. -func (*GetExecutionsByContextRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{84} +// Deprecated: Use GetContextsByIDRequest.ProtoReflect.Descriptor instead. +func (*GetContextsByIDRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{86} } -func (x *GetExecutionsByContextRequest) GetContextId() int64 { - if x != nil && x.ContextId != nil { - return *x.ContextId +func (x *GetContextsByIDRequest) GetContextIds() []int64 { + if x != nil { + return x.ContextIds } - return 0 + return nil } -func (x *GetExecutionsByContextRequest) GetOptions() *ListOperationOptions { - if x != nil { - return x.Options - } - return nil -} - -func (x *GetExecutionsByContextRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextsByIDRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -type GetExecutionsByContextResponse struct { +type GetContextsByIDResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` - // Token to use to retrieve next page of results if list options are used in - // the request. - NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` - // Options regarding transactions. - TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` + // The result is not index-aligned: if an id is not found, it is not + // returned. + Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` } -func (x *GetExecutionsByContextResponse) Reset() { - *x = GetExecutionsByContextResponse{} +func (x *GetContextsByIDResponse) Reset() { + *x = GetContextsByIDResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[85] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetExecutionsByContextResponse) String() string { +func (x *GetContextsByIDResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetExecutionsByContextResponse) ProtoMessage() {} +func (*GetContextsByIDResponse) ProtoMessage() {} -func (x *GetExecutionsByContextResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[85] +func (x *GetContextsByIDResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[87] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4958,61 +5398,45 @@ func (x *GetExecutionsByContextResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetExecutionsByContextResponse.ProtoReflect.Descriptor instead. -func (*GetExecutionsByContextResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{85} -} - -func (x *GetExecutionsByContextResponse) GetExecutions() []*Execution { - if x != nil { - return x.Executions - } - return nil -} - -func (x *GetExecutionsByContextResponse) GetNextPageToken() string { - if x != nil && x.NextPageToken != nil { - return *x.NextPageToken - } - return "" +// Deprecated: Use GetContextsByIDResponse.ProtoReflect.Descriptor instead. +func (*GetContextsByIDResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{87} } -func (x *GetExecutionsByContextResponse) GetTransactionOptions() *TransactionOptions { +func (x *GetContextsByIDResponse) GetContexts() []*Context { if x != nil { - return x.TransactionOptions + return x.Contexts } return nil } -// A lineage query request to specify the query nodes of interests and the -// boundary conditions for the returned graph. -type GetLineageGraphRequest struct { +type GetContextsByArtifactRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Options *LineageGraphQueryOptions `protobuf:"bytes,1,opt,name=options" json:"options,omitempty"` + ArtifactId *int64 `protobuf:"varint,1,opt,name=artifact_id,json=artifactId" json:"artifact_id,omitempty"` // Options regarding transactions. TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *GetLineageGraphRequest) Reset() { - *x = GetLineageGraphRequest{} +func (x *GetContextsByArtifactRequest) Reset() { + *x = GetContextsByArtifactRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[86] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetLineageGraphRequest) String() string { +func (x *GetContextsByArtifactRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetLineageGraphRequest) ProtoMessage() {} +func (*GetContextsByArtifactRequest) ProtoMessage() {} -func (x *GetLineageGraphRequest) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[86] +func (x *GetContextsByArtifactRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[88] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5023,52 +5447,50 @@ func (x *GetLineageGraphRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetLineageGraphRequest.ProtoReflect.Descriptor instead. -func (*GetLineageGraphRequest) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{86} +// Deprecated: Use GetContextsByArtifactRequest.ProtoReflect.Descriptor instead. +func (*GetContextsByArtifactRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{88} } -func (x *GetLineageGraphRequest) GetOptions() *LineageGraphQueryOptions { - if x != nil { - return x.Options +func (x *GetContextsByArtifactRequest) GetArtifactId() int64 { + if x != nil && x.ArtifactId != nil { + return *x.ArtifactId } - return nil + return 0 } -func (x *GetLineageGraphRequest) GetTransactionOptions() *TransactionOptions { +func (x *GetContextsByArtifactRequest) GetTransactionOptions() *TransactionOptions { if x != nil { return x.TransactionOptions } return nil } -// A connected lineage `subgraph` about the MLMD nodes derived from -// LineageGraphRequest.query_conditions. -type GetLineageGraphResponse struct { +type GetContextsByArtifactResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Subgraph *LineageGraph `protobuf:"bytes,1,opt,name=subgraph" json:"subgraph,omitempty"` + Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` } -func (x *GetLineageGraphResponse) Reset() { - *x = GetLineageGraphResponse{} +func (x *GetContextsByArtifactResponse) Reset() { + *x = GetContextsByArtifactResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[87] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetLineageGraphResponse) String() string { +func (x *GetContextsByArtifactResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetLineageGraphResponse) ProtoMessage() {} +func (*GetContextsByArtifactResponse) ProtoMessage() {} -func (x *GetLineageGraphResponse) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[87] +func (x *GetContextsByArtifactResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[89] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5079,54 +5501,45 @@ func (x *GetLineageGraphResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetLineageGraphResponse.ProtoReflect.Descriptor instead. -func (*GetLineageGraphResponse) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{87} +// Deprecated: Use GetContextsByArtifactResponse.ProtoReflect.Descriptor instead. +func (*GetContextsByArtifactResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{89} } -func (x *GetLineageGraphResponse) GetSubgraph() *LineageGraph { +func (x *GetContextsByArtifactResponse) GetContexts() []*Context { if x != nil { - return x.Subgraph + return x.Contexts } return nil } -type PutArtifactsRequest_Options struct { +type GetContextsByExecutionRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // When there are multiple writers to update an existing node to - // different states, there may be a race and the end result of the - // concurrent update is nondeterministic. If the field is set, then an - // optimistic concurrency control (OCC) scheme is used during update: - // it compares the `artifact`.`last_update_time_since_epoch` in the request - // with the stored `last_update_time_since_epoch` having the same - // `artifact`.`id`. If they are different, the request fails, and the user - // can read the stored node and retry node update. - // When the option is set, the timestamp after update is guaranteed to be - // increased and different from the input artifact. - // When set the option, the caller should set it for all concurrent writers. - AbortIfLatestUpdatedTimeChanged *bool `protobuf:"varint,1,opt,name=abort_if_latest_updated_time_changed,json=abortIfLatestUpdatedTimeChanged" json:"abort_if_latest_updated_time_changed,omitempty"` + ExecutionId *int64 `protobuf:"varint,1,opt,name=execution_id,json=executionId" json:"execution_id,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *PutArtifactsRequest_Options) Reset() { - *x = PutArtifactsRequest_Options{} +func (x *GetContextsByExecutionRequest) Reset() { + *x = GetContextsByExecutionRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[89] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *PutArtifactsRequest_Options) String() string { +func (x *GetContextsByExecutionRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PutArtifactsRequest_Options) ProtoMessage() {} +func (*GetContextsByExecutionRequest) ProtoMessage() {} -func (x *PutArtifactsRequest_Options) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[89] +func (x *GetContextsByExecutionRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[90] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5137,53 +5550,50 @@ func (x *PutArtifactsRequest_Options) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PutArtifactsRequest_Options.ProtoReflect.Descriptor instead. -func (*PutArtifactsRequest_Options) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{4, 0} +// Deprecated: Use GetContextsByExecutionRequest.ProtoReflect.Descriptor instead. +func (*GetContextsByExecutionRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{90} } -func (x *PutArtifactsRequest_Options) GetAbortIfLatestUpdatedTimeChanged() bool { - if x != nil && x.AbortIfLatestUpdatedTimeChanged != nil { - return *x.AbortIfLatestUpdatedTimeChanged +func (x *GetContextsByExecutionRequest) GetExecutionId() int64 { + if x != nil && x.ExecutionId != nil { + return *x.ExecutionId } - return false + return 0 } -// A pair of an artifact and an event used or generated by an execution, e.g., -// during the execution run, it uses none or many artifacts as input, and -// generate none or many artifacts as output. -type PutExecutionRequest_ArtifactAndEvent struct { +func (x *GetContextsByExecutionRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type GetContextsByExecutionResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // The pair may have an artifact. If present and no artifact.id is given, - // then it inserts the artifact, otherwise it updates the artifact. - Artifact *Artifact `protobuf:"bytes,1,opt,name=artifact" json:"artifact,omitempty"` - // The pair may have an event. Providing event.artifact_id or - // event.execution_id is optional. If the ids are given, it must align with - // the `artifact`.id / `execution`.id respectively. If artifact is not - // given and event.artifact_id is set, it must exist in the backend. - Event *Event `protobuf:"bytes,2,opt,name=event" json:"event,omitempty"` + Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` } -func (x *PutExecutionRequest_ArtifactAndEvent) Reset() { - *x = PutExecutionRequest_ArtifactAndEvent{} +func (x *GetContextsByExecutionResponse) Reset() { + *x = GetContextsByExecutionResponse{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[90] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[91] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *PutExecutionRequest_ArtifactAndEvent) String() string { +func (x *GetContextsByExecutionResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PutExecutionRequest_ArtifactAndEvent) ProtoMessage() {} +func (*GetContextsByExecutionResponse) ProtoMessage() {} -func (x *PutExecutionRequest_ArtifactAndEvent) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[90] +func (x *GetContextsByExecutionResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[91] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5194,54 +5604,45 @@ func (x *PutExecutionRequest_ArtifactAndEvent) ProtoReflect() protoreflect.Messa return mi.MessageOf(x) } -// Deprecated: Use PutExecutionRequest_ArtifactAndEvent.ProtoReflect.Descriptor instead. -func (*PutExecutionRequest_ArtifactAndEvent) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{14, 0} -} - -func (x *PutExecutionRequest_ArtifactAndEvent) GetArtifact() *Artifact { - if x != nil { - return x.Artifact - } - return nil +// Deprecated: Use GetContextsByExecutionResponse.ProtoReflect.Descriptor instead. +func (*GetContextsByExecutionResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{91} } -func (x *PutExecutionRequest_ArtifactAndEvent) GetEvent() *Event { +func (x *GetContextsByExecutionResponse) GetContexts() []*Context { if x != nil { - return x.Event + return x.Contexts } return nil } -type PutExecutionRequest_Options struct { +type GetParentContextsByContextRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // When there's a race to publish executions with a new context with the - // same context.name, by default there'll be one writer succeeds and - // the rest of the writers returning AlreadyExists errors. If set the field, - // the failed writer will reuse the stored context in the transaction. - ReuseContextIfAlreadyExist *bool `protobuf:"varint,1,opt,name=reuse_context_if_already_exist,json=reuseContextIfAlreadyExist" json:"reuse_context_if_already_exist,omitempty"` + ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` } -func (x *PutExecutionRequest_Options) Reset() { - *x = PutExecutionRequest_Options{} +func (x *GetParentContextsByContextRequest) Reset() { + *x = GetParentContextsByContextRequest{} if protoimpl.UnsafeEnabled { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[91] + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[92] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *PutExecutionRequest_Options) String() string { +func (x *GetParentContextsByContextRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PutExecutionRequest_Options) ProtoMessage() {} +func (*GetParentContextsByContextRequest) ProtoMessage() {} -func (x *PutExecutionRequest_Options) ProtoReflect() protoreflect.Message { - mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[91] +func (x *GetParentContextsByContextRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[92] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5252,314 +5653,1736 @@ func (x *PutExecutionRequest_Options) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PutExecutionRequest_Options.ProtoReflect.Descriptor instead. -func (*PutExecutionRequest_Options) Descriptor() ([]byte, []int) { - return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{14, 1} +// Deprecated: Use GetParentContextsByContextRequest.ProtoReflect.Descriptor instead. +func (*GetParentContextsByContextRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{92} } -func (x *PutExecutionRequest_Options) GetReuseContextIfAlreadyExist() bool { - if x != nil && x.ReuseContextIfAlreadyExist != nil { - return *x.ReuseContextIfAlreadyExist +func (x *GetParentContextsByContextRequest) GetContextId() int64 { + if x != nil && x.ContextId != nil { + return *x.ContextId } - return false + return 0 } -var File_ml_metadata_proto_metadata_store_service_proto protoreflect.FileDescriptor +func (x *GetParentContextsByContextRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} -var file_ml_metadata_proto_metadata_store_service_proto_rawDesc = []byte{ - 0x0a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x12, 0x0b, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x26, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x73, 0x0a, 0x0f, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x41, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x31, 0x0a, 0x08, 0x61, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x2d, 0x0a, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xbf, 0x01, 0x0a, 0x11, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x4d, 0x61, 0x70, - 0x12, 0x4e, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x4d, 0x61, 0x70, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x1a, 0x5a, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x31, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4d, 0x0a, 0x12, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x4c, 0x69, - 0x73, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xc0, 0x01, 0x0a, 0x0e, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x12, 0x3a, - 0x0a, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x48, 0x00, - 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x32, 0x0a, 0x03, 0x6d, 0x61, - 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, - 0x72, 0x75, 0x63, 0x74, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x12, 0x35, - 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, +type GetParentContextsByContextResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` +} + +func (x *GetParentContextsByContextResponse) Reset() { + *x = GetParentContextsByContextResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[93] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetParentContextsByContextResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetParentContextsByContextResponse) ProtoMessage() {} + +func (x *GetParentContextsByContextResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[93] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetParentContextsByContextResponse.ProtoReflect.Descriptor instead. +func (*GetParentContextsByContextResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{93} +} + +func (x *GetParentContextsByContextResponse) GetContexts() []*Context { + if x != nil { + return x.Contexts + } + return nil +} + +type GetChildrenContextsByContextRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetChildrenContextsByContextRequest) Reset() { + *x = GetChildrenContextsByContextRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[94] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChildrenContextsByContextRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChildrenContextsByContextRequest) ProtoMessage() {} + +func (x *GetChildrenContextsByContextRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[94] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChildrenContextsByContextRequest.ProtoReflect.Descriptor instead. +func (*GetChildrenContextsByContextRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{94} +} + +func (x *GetChildrenContextsByContextRequest) GetContextId() int64 { + if x != nil && x.ContextId != nil { + return *x.ContextId + } + return 0 +} + +func (x *GetChildrenContextsByContextRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type GetChildrenContextsByContextResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Contexts []*Context `protobuf:"bytes,1,rep,name=contexts" json:"contexts,omitempty"` +} + +func (x *GetChildrenContextsByContextResponse) Reset() { + *x = GetChildrenContextsByContextResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[95] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChildrenContextsByContextResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChildrenContextsByContextResponse) ProtoMessage() {} + +func (x *GetChildrenContextsByContextResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[95] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChildrenContextsByContextResponse.ProtoReflect.Descriptor instead. +func (*GetChildrenContextsByContextResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{95} +} + +func (x *GetChildrenContextsByContextResponse) GetContexts() []*Context { + if x != nil { + return x.Contexts + } + return nil +} + +type GetParentContextsByContextsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContextIds []int64 `protobuf:"varint,1,rep,packed,name=context_ids,json=contextIds" json:"context_ids,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetParentContextsByContextsRequest) Reset() { + *x = GetParentContextsByContextsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[96] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetParentContextsByContextsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetParentContextsByContextsRequest) ProtoMessage() {} + +func (x *GetParentContextsByContextsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[96] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetParentContextsByContextsRequest.ProtoReflect.Descriptor instead. +func (*GetParentContextsByContextsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{96} +} + +func (x *GetParentContextsByContextsRequest) GetContextIds() []int64 { + if x != nil { + return x.ContextIds + } + return nil +} + +func (x *GetParentContextsByContextsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type GetParentContextsByContextsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Contexts map[int64]*GetParentContextsByContextsResponse_ParentContextsPerChild `protobuf:"bytes,2,rep,name=contexts" json:"contexts,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (x *GetParentContextsByContextsResponse) Reset() { + *x = GetParentContextsByContextsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[97] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetParentContextsByContextsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetParentContextsByContextsResponse) ProtoMessage() {} + +func (x *GetParentContextsByContextsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[97] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetParentContextsByContextsResponse.ProtoReflect.Descriptor instead. +func (*GetParentContextsByContextsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{97} +} + +func (x *GetParentContextsByContextsResponse) GetContexts() map[int64]*GetParentContextsByContextsResponse_ParentContextsPerChild { + if x != nil { + return x.Contexts + } + return nil +} + +type GetChildrenContextsByContextsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContextIds []int64 `protobuf:"varint,1,rep,packed,name=context_ids,json=contextIds" json:"context_ids,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetChildrenContextsByContextsRequest) Reset() { + *x = GetChildrenContextsByContextsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[98] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChildrenContextsByContextsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChildrenContextsByContextsRequest) ProtoMessage() {} + +func (x *GetChildrenContextsByContextsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[98] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChildrenContextsByContextsRequest.ProtoReflect.Descriptor instead. +func (*GetChildrenContextsByContextsRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{98} +} + +func (x *GetChildrenContextsByContextsRequest) GetContextIds() []int64 { + if x != nil { + return x.ContextIds + } + return nil +} + +func (x *GetChildrenContextsByContextsRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type GetChildrenContextsByContextsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Contexts map[int64]*GetChildrenContextsByContextsResponse_ChildrenContextsPerParent `protobuf:"bytes,2,rep,name=contexts" json:"contexts,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (x *GetChildrenContextsByContextsResponse) Reset() { + *x = GetChildrenContextsByContextsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[99] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChildrenContextsByContextsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChildrenContextsByContextsResponse) ProtoMessage() {} + +func (x *GetChildrenContextsByContextsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[99] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChildrenContextsByContextsResponse.ProtoReflect.Descriptor instead. +func (*GetChildrenContextsByContextsResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{99} +} + +func (x *GetChildrenContextsByContextsResponse) GetContexts() map[int64]*GetChildrenContextsByContextsResponse_ChildrenContextsPerParent { + if x != nil { + return x.Contexts + } + return nil +} + +type GetArtifactsByContextRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` + // Specify List options. + // Currently supports: + // 1. Field to order the results. + // 2. Page size. + Options *ListOperationOptions `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetArtifactsByContextRequest) Reset() { + *x = GetArtifactsByContextRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[100] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetArtifactsByContextRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetArtifactsByContextRequest) ProtoMessage() {} + +func (x *GetArtifactsByContextRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[100] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetArtifactsByContextRequest.ProtoReflect.Descriptor instead. +func (*GetArtifactsByContextRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{100} +} + +func (x *GetArtifactsByContextRequest) GetContextId() int64 { + if x != nil && x.ContextId != nil { + return *x.ContextId + } + return 0 +} + +func (x *GetArtifactsByContextRequest) GetOptions() *ListOperationOptions { + if x != nil { + return x.Options + } + return nil +} + +func (x *GetArtifactsByContextRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type GetArtifactsByContextResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Artifacts []*Artifact `protobuf:"bytes,1,rep,name=artifacts" json:"artifacts,omitempty"` + // Token to use to retrieve next page of results if list options are used in + // the request. + NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` +} + +func (x *GetArtifactsByContextResponse) Reset() { + *x = GetArtifactsByContextResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[101] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetArtifactsByContextResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetArtifactsByContextResponse) ProtoMessage() {} + +func (x *GetArtifactsByContextResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[101] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetArtifactsByContextResponse.ProtoReflect.Descriptor instead. +func (*GetArtifactsByContextResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{101} +} + +func (x *GetArtifactsByContextResponse) GetArtifacts() []*Artifact { + if x != nil { + return x.Artifacts + } + return nil +} + +func (x *GetArtifactsByContextResponse) GetNextPageToken() string { + if x != nil && x.NextPageToken != nil { + return *x.NextPageToken + } + return "" +} + +type GetExecutionsByContextRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ContextId *int64 `protobuf:"varint,1,opt,name=context_id,json=contextId" json:"context_id,omitempty"` + // Specify List options. + // Currently supports: + // 1. Field to order the results. + // 2. Page size. + Options *ListOperationOptions `protobuf:"bytes,2,opt,name=options" json:"options,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetExecutionsByContextRequest) Reset() { + *x = GetExecutionsByContextRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[102] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetExecutionsByContextRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetExecutionsByContextRequest) ProtoMessage() {} + +func (x *GetExecutionsByContextRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[102] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetExecutionsByContextRequest.ProtoReflect.Descriptor instead. +func (*GetExecutionsByContextRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{102} +} + +func (x *GetExecutionsByContextRequest) GetContextId() int64 { + if x != nil && x.ContextId != nil { + return *x.ContextId + } + return 0 +} + +func (x *GetExecutionsByContextRequest) GetOptions() *ListOperationOptions { + if x != nil { + return x.Options + } + return nil +} + +func (x *GetExecutionsByContextRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type GetExecutionsByContextResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Executions []*Execution `protobuf:"bytes,1,rep,name=executions" json:"executions,omitempty"` + // Token to use to retrieve next page of results if list options are used in + // the request. + NextPageToken *string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken" json:"next_page_token,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,3,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetExecutionsByContextResponse) Reset() { + *x = GetExecutionsByContextResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[103] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetExecutionsByContextResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetExecutionsByContextResponse) ProtoMessage() {} + +func (x *GetExecutionsByContextResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[103] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetExecutionsByContextResponse.ProtoReflect.Descriptor instead. +func (*GetExecutionsByContextResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{103} +} + +func (x *GetExecutionsByContextResponse) GetExecutions() []*Execution { + if x != nil { + return x.Executions + } + return nil +} + +func (x *GetExecutionsByContextResponse) GetNextPageToken() string { + if x != nil && x.NextPageToken != nil { + return *x.NextPageToken + } + return "" +} + +func (x *GetExecutionsByContextResponse) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +// TODO(b/283852485): Deprecate GetLineageGraph API after migration to +// GetLineageSubgraph API. +// A lineage query request to specify the query nodes of interest and the +// boundary conditions for pruning the returned graph. +type GetLineageGraphRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Options *LineageGraphQueryOptions `protobuf:"bytes,1,opt,name=options" json:"options,omitempty"` + // Options regarding transactions. + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetLineageGraphRequest) Reset() { + *x = GetLineageGraphRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[104] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetLineageGraphRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetLineageGraphRequest) ProtoMessage() {} + +func (x *GetLineageGraphRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[104] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetLineageGraphRequest.ProtoReflect.Descriptor instead. +func (*GetLineageGraphRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{104} +} + +func (x *GetLineageGraphRequest) GetOptions() *LineageGraphQueryOptions { + if x != nil { + return x.Options + } + return nil +} + +func (x *GetLineageGraphRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +// A connected lineage `subgraph` about the MLMD nodes derived from +// LineageGraphRequest.query_conditions. +type GetLineageGraphResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Subgraph *LineageGraph `protobuf:"bytes,1,opt,name=subgraph" json:"subgraph,omitempty"` +} + +func (x *GetLineageGraphResponse) Reset() { + *x = GetLineageGraphResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[105] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetLineageGraphResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetLineageGraphResponse) ProtoMessage() {} + +func (x *GetLineageGraphResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[105] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetLineageGraphResponse.ProtoReflect.Descriptor instead. +func (*GetLineageGraphResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{105} +} + +func (x *GetLineageGraphResponse) GetSubgraph() *LineageGraph { + if x != nil { + return x.Subgraph + } + return nil +} + +type GetLineageSubgraphRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Query options for lineage graph tracing from a list of interested + // nodes. + // A lineage subgraph without node details (e.g., external_id, properties) + // will be returned. Please refer to `LineageSubgraphQueryOptions` for more + // details. + LineageSubgraphQueryOptions *LineageSubgraphQueryOptions `protobuf:"bytes,1,opt,name=lineage_subgraph_query_options,json=lineageSubgraphQueryOptions" json:"lineage_subgraph_query_options,omitempty"` + // `read_mask` contains user specified paths of fields that should be included + // in the returned lineage subgraph. + // Supported field paths are: 'artifacts', 'executions', 'contexts', + // 'artifact_types', 'execution_types', 'context_types', and 'events'. + // TODO(b/283852485): Include 'associations' or 'attributions' in the + // returned graph. + // If 'artifacts', 'executions', or 'contexts' is specified in `read_mask`, + // the dehydrated nodes will be included. + // Note: A dehydrated node means a node containing only its id and no + // other information. User should call GetNodesByID or other APIs to get + // node details later on. + // If 'artifact_types', 'execution_types', or 'context_types' is specified + // in `read_mask`, all the node types will be included. + // If 'events' is specified in `read_mask`, the events will be included. + // If `read_mask` is not set, the API will return all the fields in + // the returned graph. + // Note: Only paths of fields in LineageGraph message are supported. Paths + // of fields in the submessage, such as "artifacts.id", "contexts.name" are + // not acknowledged. + ReadMask *fieldmaskpb.FieldMask `protobuf:"bytes,3,opt,name=read_mask,json=readMask" json:"read_mask,omitempty"` + TransactionOptions *TransactionOptions `protobuf:"bytes,2,opt,name=transaction_options,json=transactionOptions" json:"transaction_options,omitempty"` +} + +func (x *GetLineageSubgraphRequest) Reset() { + *x = GetLineageSubgraphRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[106] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetLineageSubgraphRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetLineageSubgraphRequest) ProtoMessage() {} + +func (x *GetLineageSubgraphRequest) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[106] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetLineageSubgraphRequest.ProtoReflect.Descriptor instead. +func (*GetLineageSubgraphRequest) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{106} +} + +func (x *GetLineageSubgraphRequest) GetLineageSubgraphQueryOptions() *LineageSubgraphQueryOptions { + if x != nil { + return x.LineageSubgraphQueryOptions + } + return nil +} + +func (x *GetLineageSubgraphRequest) GetReadMask() *fieldmaskpb.FieldMask { + if x != nil { + return x.ReadMask + } + return nil +} + +func (x *GetLineageSubgraphRequest) GetTransactionOptions() *TransactionOptions { + if x != nil { + return x.TransactionOptions + } + return nil +} + +type GetLineageSubgraphResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // A lineage subgraph of MLMD nodes and relations retrieved from lineage + // graph tracing. + LineageSubgraph *LineageGraph `protobuf:"bytes,1,opt,name=lineage_subgraph,json=lineageSubgraph" json:"lineage_subgraph,omitempty"` +} + +func (x *GetLineageSubgraphResponse) Reset() { + *x = GetLineageSubgraphResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[107] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetLineageSubgraphResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetLineageSubgraphResponse) ProtoMessage() {} + +func (x *GetLineageSubgraphResponse) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[107] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetLineageSubgraphResponse.ProtoReflect.Descriptor instead. +func (*GetLineageSubgraphResponse) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{107} +} + +func (x *GetLineageSubgraphResponse) GetLineageSubgraph() *LineageGraph { + if x != nil { + return x.LineageSubgraph + } + return nil +} + +type PutArtifactsRequest_Options struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // When there are multiple writers to update an existing node to + // different states, there may be a race and the end result of the + // concurrent update is nondeterministic. If the field is set, then an + // optimistic concurrency control (OCC) scheme is used during update: + // it compares the `artifact`.`last_update_time_since_epoch` in the request + // with the stored `last_update_time_since_epoch` having the same + // `artifact`.`id`. If they are different, the request fails, and the user + // can read the stored node and retry node update. + // When the option is set, the timestamp after update is guaranteed to be + // increased and different from the input artifact. + // When set the option, the caller should set it for all concurrent writers. + AbortIfLatestUpdatedTimeChanged *bool `protobuf:"varint,1,opt,name=abort_if_latest_updated_time_changed,json=abortIfLatestUpdatedTimeChanged" json:"abort_if_latest_updated_time_changed,omitempty"` +} + +func (x *PutArtifactsRequest_Options) Reset() { + *x = PutArtifactsRequest_Options{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[109] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutArtifactsRequest_Options) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutArtifactsRequest_Options) ProtoMessage() {} + +func (x *PutArtifactsRequest_Options) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[109] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutArtifactsRequest_Options.ProtoReflect.Descriptor instead. +func (*PutArtifactsRequest_Options) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{4, 0} +} + +func (x *PutArtifactsRequest_Options) GetAbortIfLatestUpdatedTimeChanged() bool { + if x != nil && x.AbortIfLatestUpdatedTimeChanged != nil { + return *x.AbortIfLatestUpdatedTimeChanged + } + return false +} + +// A pair of an artifact and an event used or generated by an execution, e.g., +// during the execution run, it uses none or many artifacts as input, and +// generate none or many artifacts as output. +type PutExecutionRequest_ArtifactAndEvent struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The pair may have an artifact. If present and no artifact.id is given, + // then it inserts the artifact, otherwise it updates the artifact. + Artifact *Artifact `protobuf:"bytes,1,opt,name=artifact" json:"artifact,omitempty"` + // The pair may have an event. Providing event.artifact_id or + // event.execution_id is optional. If the ids are given, it must align with + // the `artifact`.id / `execution`.id respectively. If artifact is not + // given and event.artifact_id is set, it must exist in the backend. + Event *Event `protobuf:"bytes,2,opt,name=event" json:"event,omitempty"` +} + +func (x *PutExecutionRequest_ArtifactAndEvent) Reset() { + *x = PutExecutionRequest_ArtifactAndEvent{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[110] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutExecutionRequest_ArtifactAndEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutExecutionRequest_ArtifactAndEvent) ProtoMessage() {} + +func (x *PutExecutionRequest_ArtifactAndEvent) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[110] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutExecutionRequest_ArtifactAndEvent.ProtoReflect.Descriptor instead. +func (*PutExecutionRequest_ArtifactAndEvent) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{14, 0} +} + +func (x *PutExecutionRequest_ArtifactAndEvent) GetArtifact() *Artifact { + if x != nil { + return x.Artifact + } + return nil +} + +func (x *PutExecutionRequest_ArtifactAndEvent) GetEvent() *Event { + if x != nil { + return x.Event + } + return nil +} + +type PutExecutionRequest_Options struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // When there's a race to publish executions with a new context with the + // same context.name, by default there'll be one writer succeeds and + // the rest of the writers returning AlreadyExists errors. If set to true, + // the API will reuse the stored context in the transaction and perform + // an update. + ReuseContextIfAlreadyExist *bool `protobuf:"varint,1,opt,name=reuse_context_if_already_exist,json=reuseContextIfAlreadyExist" json:"reuse_context_if_already_exist,omitempty"` + // When there's a race to publish executions with a new artifact with the + // same artifact.external_id, by default there'll be one writer succeeds and + // the rest of the writers returning AlreadyExists errors. + // If set to true and an Artifact has non-empty external_id, + // the API will reuse the stored artifact in the transaction and + // perform an update. Otherwise, it will fall back to relying on `id` field + // to decide if it's update (if `id` exists) or insert (if `id` is empty). + ReuseArtifactIfAlreadyExistByExternalId *bool `protobuf:"varint,2,opt,name=reuse_artifact_if_already_exist_by_external_id,json=reuseArtifactIfAlreadyExistByExternalId" json:"reuse_artifact_if_already_exist_by_external_id,omitempty"` +} + +func (x *PutExecutionRequest_Options) Reset() { + *x = PutExecutionRequest_Options{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[111] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutExecutionRequest_Options) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutExecutionRequest_Options) ProtoMessage() {} + +func (x *PutExecutionRequest_Options) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[111] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutExecutionRequest_Options.ProtoReflect.Descriptor instead. +func (*PutExecutionRequest_Options) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{14, 1} +} + +func (x *PutExecutionRequest_Options) GetReuseContextIfAlreadyExist() bool { + if x != nil && x.ReuseContextIfAlreadyExist != nil { + return *x.ReuseContextIfAlreadyExist + } + return false +} + +func (x *PutExecutionRequest_Options) GetReuseArtifactIfAlreadyExistByExternalId() bool { + if x != nil && x.ReuseArtifactIfAlreadyExistByExternalId != nil { + return *x.ReuseArtifactIfAlreadyExistByExternalId + } + return false +} + +type PutLineageSubgraphRequest_EventEdge struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Index in the array of executions. + ExecutionIndex *int32 `protobuf:"varint,1,opt,name=execution_index,json=executionIndex" json:"execution_index,omitempty"` + // Index in the array of artifacts. + ArtifactIndex *int32 `protobuf:"varint,2,opt,name=artifact_index,json=artifactIndex" json:"artifact_index,omitempty"` + Event *Event `protobuf:"bytes,3,opt,name=event" json:"event,omitempty"` +} + +func (x *PutLineageSubgraphRequest_EventEdge) Reset() { + *x = PutLineageSubgraphRequest_EventEdge{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[112] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutLineageSubgraphRequest_EventEdge) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutLineageSubgraphRequest_EventEdge) ProtoMessage() {} + +func (x *PutLineageSubgraphRequest_EventEdge) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[112] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutLineageSubgraphRequest_EventEdge.ProtoReflect.Descriptor instead. +func (*PutLineageSubgraphRequest_EventEdge) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{16, 0} +} + +func (x *PutLineageSubgraphRequest_EventEdge) GetExecutionIndex() int32 { + if x != nil && x.ExecutionIndex != nil { + return *x.ExecutionIndex + } + return 0 +} + +func (x *PutLineageSubgraphRequest_EventEdge) GetArtifactIndex() int32 { + if x != nil && x.ArtifactIndex != nil { + return *x.ArtifactIndex + } + return 0 +} + +func (x *PutLineageSubgraphRequest_EventEdge) GetEvent() *Event { + if x != nil { + return x.Event + } + return nil +} + +type PutLineageSubgraphRequest_Options struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // When there's a race to publish executions with a new context with the + // same context.name, by default there'll be one writer succeeds and + // the rest of the writers returning AlreadyExists errors. If set to true, + // the API will reuse the stored context in the transaction and perform + // an update. + ReuseContextIfAlreadyExist *bool `protobuf:"varint,1,opt,name=reuse_context_if_already_exist,json=reuseContextIfAlreadyExist" json:"reuse_context_if_already_exist,omitempty"` + // When there's a race to publish executions with a new artifact with the + // same artifact.external_id, by default there'll be one writer succeeds and + // the rest of the writers returning AlreadyExists errors. + // If set to true and an Artifact has non-empty external_id, + // the API will reuse the stored artifact in the transaction and + // perform an update. Otherwise, it will fall back to relying on `id` field + // to decide if it's update (if `id` exists) or insert (if `id` is empty). + ReuseArtifactIfAlreadyExistByExternalId *bool `protobuf:"varint,2,opt,name=reuse_artifact_if_already_exist_by_external_id,json=reuseArtifactIfAlreadyExistByExternalId" json:"reuse_artifact_if_already_exist_by_external_id,omitempty"` +} + +func (x *PutLineageSubgraphRequest_Options) Reset() { + *x = PutLineageSubgraphRequest_Options{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[113] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PutLineageSubgraphRequest_Options) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PutLineageSubgraphRequest_Options) ProtoMessage() {} + +func (x *PutLineageSubgraphRequest_Options) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[113] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PutLineageSubgraphRequest_Options.ProtoReflect.Descriptor instead. +func (*PutLineageSubgraphRequest_Options) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{16, 1} +} + +func (x *PutLineageSubgraphRequest_Options) GetReuseContextIfAlreadyExist() bool { + if x != nil && x.ReuseContextIfAlreadyExist != nil { + return *x.ReuseContextIfAlreadyExist + } + return false +} + +func (x *PutLineageSubgraphRequest_Options) GetReuseArtifactIfAlreadyExistByExternalId() bool { + if x != nil && x.ReuseArtifactIfAlreadyExistByExternalId != nil { + return *x.ReuseArtifactIfAlreadyExistByExternalId + } + return false +} + +type GetParentContextsByContextsResponse_ParentContextsPerChild struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ParentContexts []*Context `protobuf:"bytes,1,rep,name=parent_contexts,json=parentContexts" json:"parent_contexts,omitempty"` +} + +func (x *GetParentContextsByContextsResponse_ParentContextsPerChild) Reset() { + *x = GetParentContextsByContextsResponse_ParentContextsPerChild{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[114] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetParentContextsByContextsResponse_ParentContextsPerChild) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetParentContextsByContextsResponse_ParentContextsPerChild) ProtoMessage() {} + +func (x *GetParentContextsByContextsResponse_ParentContextsPerChild) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[114] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetParentContextsByContextsResponse_ParentContextsPerChild.ProtoReflect.Descriptor instead. +func (*GetParentContextsByContextsResponse_ParentContextsPerChild) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{97, 0} +} + +func (x *GetParentContextsByContextsResponse_ParentContextsPerChild) GetParentContexts() []*Context { + if x != nil { + return x.ParentContexts + } + return nil +} + +type GetChildrenContextsByContextsResponse_ChildrenContextsPerParent struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ChildrenContexts []*Context `protobuf:"bytes,1,rep,name=children_contexts,json=childrenContexts" json:"children_contexts,omitempty"` +} + +func (x *GetChildrenContextsByContextsResponse_ChildrenContextsPerParent) Reset() { + *x = GetChildrenContextsByContextsResponse_ChildrenContextsPerParent{} + if protoimpl.UnsafeEnabled { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[116] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetChildrenContextsByContextsResponse_ChildrenContextsPerParent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetChildrenContextsByContextsResponse_ChildrenContextsPerParent) ProtoMessage() {} + +func (x *GetChildrenContextsByContextsResponse_ChildrenContextsPerParent) ProtoReflect() protoreflect.Message { + mi := &file_ml_metadata_proto_metadata_store_service_proto_msgTypes[116] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetChildrenContextsByContextsResponse_ChildrenContextsPerParent.ProtoReflect.Descriptor instead. +func (*GetChildrenContextsByContextsResponse_ChildrenContextsPerParent) Descriptor() ([]byte, []int) { + return file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP(), []int{99, 0} +} + +func (x *GetChildrenContextsByContextsResponse_ChildrenContextsPerParent) GetChildrenContexts() []*Context { + if x != nil { + return x.ChildrenContexts + } + return nil +} + +var File_ml_metadata_proto_metadata_store_service_proto protoreflect.FileDescriptor + +var file_ml_metadata_proto_metadata_store_service_proto_rawDesc = []byte{ + 0x0a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x0b, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x20, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x26, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x73, 0x0a, 0x0f, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x41, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x31, 0x0a, 0x08, 0x61, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, - 0x04, 0x6c, 0x69, 0x73, 0x74, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xe8, - 0x01, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x61, 0x63, 0x74, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x2d, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xbf, 0x01, 0x0a, + 0x11, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x4d, + 0x61, 0x70, 0x12, 0x4e, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x4d, 0x61, 0x70, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, + 0x65, 0x73, 0x1a, 0x5a, 0x0a, 0x0f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x31, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4d, + 0x0a, 0x12, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x4c, 0x69, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x52, 0x08, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xc0, 0x01, + 0x0a, 0x0e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x12, 0x3a, 0x0a, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x6e, 0x64, 0x54, 0x79, 0x70, 0x65, + 0x48, 0x00, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x32, 0x0a, 0x03, + 0x6d, 0x61, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x42, 0x0a, 0x07, 0x6f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, - 0x58, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4d, 0x0a, 0x24, 0x61, 0x62, - 0x6f, 0x72, 0x74, 0x5f, 0x69, 0x66, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x6e, 0x67, - 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1f, 0x61, 0x62, 0x6f, 0x72, 0x74, 0x49, - 0x66, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, - 0x6d, 0x65, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x22, 0x39, 0x0a, 0x14, 0x50, 0x75, 0x74, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x49, 0x64, 0x73, 0x22, 0x8a, 0x02, 0x0a, 0x16, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3e, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x24, 0x0a, 0x0e, 0x63, 0x61, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x61, 0x6e, 0x41, 0x64, 0x64, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x6e, 0x5f, 0x6f, 0x6d, 0x69, - 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, - 0x63, 0x61, 0x6e, 0x4f, 0x6d, 0x69, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x2e, 0x0a, - 0x11, 0x63, 0x61, 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0f, 0x63, 0x61, - 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x32, 0x0a, - 0x10, 0x61, 0x6c, 0x6c, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, - 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x42, 0x02, 0x18, - 0x01, 0x52, 0x0e, 0x61, 0x6c, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x4d, 0x61, 0x74, 0x63, - 0x68, 0x22, 0x32, 0x0a, 0x17, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, - 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, - 0x79, 0x70, 0x65, 0x49, 0x64, 0x22, 0x4e, 0x0a, 0x14, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x36, 0x0a, - 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3c, 0x0a, 0x15, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, - 0x0a, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x73, 0x22, 0x8e, 0x02, 0x0a, 0x17, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x41, 0x0a, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x61, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x5f, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x61, 0x6e, 0x41, - 0x64, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x6e, 0x5f, - 0x6f, 0x6d, 0x69, 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0d, 0x63, 0x61, 0x6e, 0x4f, 0x6d, 0x69, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x12, 0x2e, 0x0a, 0x11, 0x63, 0x61, 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, - 0x0f, 0x63, 0x61, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x12, 0x32, 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x5f, 0x6d, - 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, - 0x42, 0x02, 0x18, 0x01, 0x52, 0x0e, 0x61, 0x6c, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x22, 0x33, 0x0a, 0x18, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x17, 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x06, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x22, 0x3e, 0x0a, 0x10, 0x50, 0x75, 0x74, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, - 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x50, 0x75, 0x74, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xe6, - 0x03, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x63, 0x0a, 0x14, - 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x70, - 0x61, 0x69, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x12, 0x61, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, - 0x73, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x12, 0x42, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x6f, 0x0a, 0x10, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x41, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x08, 0x61, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x28, - 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x1a, 0x4d, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x42, 0x0a, 0x1e, 0x72, 0x65, 0x75, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x66, 0x5f, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x5f, - 0x65, 0x78, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1a, 0x72, 0x65, 0x75, - 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x66, 0x41, 0x6c, 0x72, 0x65, 0x61, - 0x64, 0x79, 0x45, 0x78, 0x69, 0x73, 0x74, 0x22, 0x7d, 0x0a, 0x14, 0x50, 0x75, 0x74, 0x45, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, - 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x49, 0x64, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x49, 0x64, 0x73, 0x22, 0x89, 0x03, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x0e, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x61, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0f, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, - 0x73, 0x12, 0x3d, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x03, 0x6d, 0x61, 0x70, + 0x12, 0x35, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x48, + 0x00, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x22, 0xf7, 0x02, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x42, 0x0a, + 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x3b, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x6d, 0x61, + 0x73, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x4d, 0x61, 0x73, 0x6b, 0x52, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x73, 0x6b, + 0x1a, 0x58, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4d, 0x0a, 0x24, 0x61, + 0x62, 0x6f, 0x72, 0x74, 0x5f, 0x69, 0x66, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x63, 0x68, 0x61, 0x6e, + 0x67, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1f, 0x61, 0x62, 0x6f, 0x72, 0x74, + 0x49, 0x66, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, + 0x69, 0x6d, 0x65, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x64, 0x22, 0x39, 0x0a, 0x14, 0x50, 0x75, + 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, + 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x49, 0x64, 0x73, 0x22, 0xdc, 0x02, 0x0a, 0x16, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x3e, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x61, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x5f, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x61, 0x6e, 0x41, 0x64, 0x64, + 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x61, 0x6e, 0x41, 0x64, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x6e, 0x5f, 0x6f, 0x6d, - 0x69, 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x69, 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x63, 0x61, 0x6e, 0x4f, 0x6d, 0x69, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x2e, 0x0a, 0x11, 0x63, 0x61, 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0f, 0x63, + 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0f, 0x63, 0x61, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x5f, 0x6d, 0x61, 0x74, - 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x42, 0x02, + 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0e, 0x61, 0x6c, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x4d, 0x61, 0x74, - 0x63, 0x68, 0x22, 0x96, 0x01, 0x0a, 0x10, 0x50, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x61, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x03, 0x52, 0x0f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x49, 0x64, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x03, 0x52, - 0x10, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x49, 0x64, - 0x73, 0x12, 0x28, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0e, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x22, 0x86, 0x02, 0x0a, 0x15, - 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, + 0x63, 0x68, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, + 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x32, 0x0a, 0x17, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x17, 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x06, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x22, 0xdd, 0x01, 0x0a, 0x14, 0x50, 0x75, 0x74, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3b, 0x0a, 0x0b, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x73, 0x6b, 0x52, 0x0a, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x73, 0x6b, 0x22, 0x3c, 0x0a, 0x15, 0x50, 0x75, 0x74, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x22, 0xe0, 0x02, 0x0a, 0x17, 0x50, 0x75, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x41, 0x0a, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x61, 0x6e, 0x5f, 0x61, 0x64, 0x64, + 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, + 0x61, 0x6e, 0x41, 0x64, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x63, + 0x61, 0x6e, 0x5f, 0x6f, 0x6d, 0x69, 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x63, 0x61, 0x6e, 0x4f, 0x6d, 0x69, 0x74, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x12, 0x2e, 0x0a, 0x11, 0x63, 0x61, 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, + 0x18, 0x01, 0x52, 0x0f, 0x63, 0x61, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, + 0x72, 0x75, 0x65, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0e, 0x61, 0x6c, 0x6c, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x33, 0x0a, 0x18, 0x50, 0x75, 0x74, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x22, 0x90, + 0x01, 0x0a, 0x10, 0x50, 0x75, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, + 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x22, 0x13, 0x0a, 0x11, 0x50, 0x75, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x9a, 0x05, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, + 0x0a, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x63, 0x0a, 0x14, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x61, 0x69, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x6e, 0x64, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x12, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x50, 0x61, 0x69, 0x72, 0x73, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x61, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x5f, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x61, 0x6e, 0x41, - 0x64, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x6e, 0x5f, - 0x6f, 0x6d, 0x69, 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0d, 0x63, 0x61, 0x6e, 0x4f, 0x6d, 0x69, 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x12, 0x2e, 0x0a, 0x11, 0x63, 0x61, 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, - 0x0f, 0x63, 0x61, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x12, 0x32, 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x5f, 0x6d, - 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, - 0x42, 0x02, 0x18, 0x01, 0x52, 0x0e, 0x61, 0x6c, 0x6c, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x4d, - 0x61, 0x74, 0x63, 0x68, 0x22, 0x31, 0x0a, 0x16, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, - 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x06, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x22, 0x46, 0x0a, 0x12, 0x50, 0x75, 0x74, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x30, 0x0a, - 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x42, 0x0a, 0x07, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x1a, 0x6f, 0x0a, 0x10, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x6e, 0x64, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x08, + 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x28, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x1a, 0xae, 0x01, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x42, + 0x0a, 0x1e, 0x72, 0x65, 0x75, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, + 0x69, 0x66, 0x5f, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x5f, 0x65, 0x78, 0x69, 0x73, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1a, 0x72, 0x65, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x49, 0x66, 0x41, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x45, 0x78, 0x69, + 0x73, 0x74, 0x12, 0x5f, 0x0a, 0x2e, 0x72, 0x65, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x66, 0x5f, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x5f, + 0x65, 0x78, 0x69, 0x73, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x27, 0x72, 0x65, 0x75, 0x73, + 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x66, 0x41, 0x6c, 0x72, 0x65, 0x61, + 0x64, 0x79, 0x45, 0x78, 0x69, 0x73, 0x74, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x49, 0x64, 0x22, 0x7d, 0x0a, 0x14, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x21, + 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, + 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, + 0x64, 0x73, 0x22, 0xe2, 0x05, 0x0a, 0x19, 0x50, 0x75, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, + 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x30, 0x0a, + 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, - 0x36, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0a, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x73, 0x22, 0xa3, 0x01, 0x0a, 0x25, 0x50, 0x75, 0x74, 0x41, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, + 0x51, 0x0a, 0x0b, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x64, 0x67, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, + 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x45, 0x64, 0x67, 0x65, 0x52, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x45, 0x64, 0x67, + 0x65, 0x73, 0x12, 0x48, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x50, 0x75, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, + 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x85, + 0x01, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x45, 0x64, 0x67, 0x65, 0x12, 0x27, 0x0a, 0x0f, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x25, 0x0a, 0x0e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x61, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x28, 0x0a, 0x05, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, + 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x1a, 0xae, 0x01, 0x0a, 0x07, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x42, 0x0a, 0x1e, 0x72, 0x65, 0x75, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x5f, 0x69, 0x66, 0x5f, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x5f, 0x65, + 0x78, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x1a, 0x72, 0x65, 0x75, 0x73, + 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x66, 0x41, 0x6c, 0x72, 0x65, 0x61, 0x64, + 0x79, 0x45, 0x78, 0x69, 0x73, 0x74, 0x12, 0x5f, 0x0a, 0x2e, 0x72, 0x65, 0x75, 0x73, 0x65, 0x5f, + 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x66, 0x5f, 0x61, 0x6c, 0x72, 0x65, + 0x61, 0x64, 0x79, 0x5f, 0x65, 0x78, 0x69, 0x73, 0x74, 0x5f, 0x62, 0x79, 0x5f, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x27, + 0x72, 0x65, 0x75, 0x73, 0x65, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x66, 0x41, + 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x45, 0x78, 0x69, 0x73, 0x74, 0x42, 0x79, 0x45, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x22, 0x91, 0x01, 0x0a, 0x1a, 0x50, 0x75, 0x74, 0x4c, + 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x42, 0x02, 0x10, + 0x01, 0x52, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x12, + 0x25, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x03, 0x42, 0x02, 0x10, 0x01, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x49, 0x64, 0x73, 0x12, 0x23, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x42, 0x02, 0x10, 0x01, 0x52, + 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x73, 0x22, 0xdb, 0x03, 0x0a, 0x0f, + 0x50, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x40, 0x0a, 0x0e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x73, 0x12, 0x43, 0x0a, 0x0f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x61, 0x6e, 0x5f, 0x61, 0x64, 0x64, + 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, + 0x61, 0x6e, 0x41, 0x64, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x63, + 0x61, 0x6e, 0x5f, 0x6f, 0x6d, 0x69, 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x63, 0x61, 0x6e, 0x4f, 0x6d, 0x69, 0x74, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x12, 0x2e, 0x0a, 0x11, 0x63, 0x61, 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, + 0x18, 0x01, 0x52, 0x0f, 0x63, 0x61, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, + 0x72, 0x75, 0x65, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0e, 0x61, 0x6c, 0x6c, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x10, 0x50, 0x75, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, + 0x0a, 0x11, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, + 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0f, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x03, 0x52, 0x10, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x03, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x49, + 0x64, 0x73, 0x22, 0xd8, 0x02, 0x0a, 0x15, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x0c, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x63, 0x61, 0x6e, + 0x5f, 0x61, 0x64, 0x64, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0c, 0x63, 0x61, 0x6e, 0x41, 0x64, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, + 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x6e, 0x5f, 0x6f, 0x6d, 0x69, 0x74, 0x5f, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x63, 0x61, 0x6e, 0x4f, 0x6d, 0x69, + 0x74, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x2e, 0x0a, 0x11, 0x63, 0x61, 0x6e, 0x5f, 0x64, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0f, 0x63, 0x61, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x10, 0x61, 0x6c, 0x6c, 0x5f, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0e, 0x61, 0x6c, 0x6c, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, 0x50, 0x0a, 0x13, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x31, 0x0a, + 0x16, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x79, 0x70, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, + 0x22, 0xd5, 0x01, 0x0a, 0x12, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, + 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3b, 0x0a, 0x0b, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x73, 0x6b, 0x52, 0x0a, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x73, 0x6b, 0x22, 0x36, 0x0a, 0x13, 0x50, 0x75, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x03, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x73, + 0x22, 0xf5, 0x01, 0x0a, 0x25, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x0c, 0x61, 0x74, + 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, + 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x61, 0x74, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3c, 0x0a, 0x0c, 0x61, 0x73, 0x73, 0x6f, + 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x73, 0x73, + 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x61, 0x73, 0x73, 0x6f, 0x63, 0x69, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, 0x0a, 0x26, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, - 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3c, 0x0a, 0x0c, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x0c, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x3c, 0x0a, 0x0c, 0x61, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x0c, 0x61, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x28, 0x0a, - 0x26, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x5f, 0x0a, 0x18, 0x50, 0x75, 0x74, 0x50, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x61, 0x72, 0x65, 0x6e, - 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x0e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x1b, 0x0a, 0x19, 0x50, 0x75, 0x74, 0x50, - 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, - 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0x79, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, - 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xd8, 0x01, - 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, - 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0xb1, 0x01, 0x0a, 0x18, 0x50, 0x75, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x43, 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x52, 0x0e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x1b, 0x0a, 0x19, 0x50, 0x75, 0x74, 0x50, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0xea, 0x01, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, - 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x55, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, - 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x08, - 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x22, - 0x8e, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, - 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x61, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x03, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x73, 0x12, 0x50, + 0x6e, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x4f, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, - 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x09, - 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x22, 0x79, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, + 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, + 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, + 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, + 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xd8, 0x01, 0x0a, 0x1f, + 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, + 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x23, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x55, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, + 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x08, 0x61, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x22, 0xcd, 0x01, + 0x0a, 0x17, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, + 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, + 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x73, 0x12, 0x3d, 0x0a, 0x17, + 0x70, 0x6f, 0x70, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x5f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, + 0x61, 0x6c, 0x73, 0x65, 0x52, 0x15, 0x70, 0x6f, 0x70, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x91, 0x01, + 0x0a, 0x18, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, + 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, + 0x40, 0x0a, 0x0e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0xa4, 0x01, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, @@ -5664,638 +7487,903 @@ var file_ml_metadata_proto_metadata_store_service_proto_rawDesc = []byte{ 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x73, 0x22, 0xeb, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, - 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, - 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x7d, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, - 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, - 0xdb, 0x01, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x50, 0x0a, 0x13, 0x74, - 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x59, 0x0a, - 0x21, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x54, - 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x34, 0x0a, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x91, 0x01, 0x0a, 0x18, 0x47, 0x65, 0x74, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0c, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, + 0x65, 0x73, 0x22, 0x97, 0x01, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x53, 0x0a, 0x19, - 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, - 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x22, 0xab, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, - 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, - 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, - 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, - 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, - 0x5d, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0x97, - 0x01, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, - 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x58, 0x0a, 0x21, + 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x22, 0x98, 0x01, 0x0a, 0x21, 0x47, 0x65, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, + 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, + 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x22, 0x5c, 0x0a, 0x22, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, + 0x96, 0x01, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, + 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, + 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4d, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x44, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x94, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, - 0x44, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, - 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4c, - 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x8a, 0x01, 0x0a, - 0x1b, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, - 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x07, - 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x60, 0x0a, 0x1c, 0x47, 0x65, 0x74, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, - 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x61, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x1c, - 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, - 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, - 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x07, - 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x64, 0x0a, 0x1d, 0x47, 0x65, 0x74, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, - 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x0f, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, - 0xa9, 0x01, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, - 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, - 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, - 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, - 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x55, 0x0a, 0x16, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x22, 0x89, 0x01, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x03, 0x52, 0x07, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x5c, - 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, - 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0xa3, 0x01, 0x0a, - 0x12, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x54, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x9b, + 0x01, 0x0a, 0x24, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x69, 0x0a, 0x25, + 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x9c, 0x01, 0x0a, 0x25, 0x47, 0x65, 0x74, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x6d, 0x0a, 0x26, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x43, 0x0a, 0x0f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0x9a, 0x01, 0x0a, 0x23, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, + 0x0c, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0x6f, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, - 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x22, 0xe9, 0x01, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x3b, 0x0a, - 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, - 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, - 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, - 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, - 0x75, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x26, + 0x6e, 0x73, 0x22, 0x65, 0x0a, 0x24, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x0d, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0xeb, 0x01, 0x0a, 0x1a, 0x47, 0x65, + 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, + 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x7d, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, - 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xd5, 0x01, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, - 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, - 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, - 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, - 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x50, 0x0a, 0x13, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x51, - 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x79, 0x54, 0x79, - 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x2e, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x03, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, + 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xdb, 0x01, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, + 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, + 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, + 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x59, 0x0a, 0x21, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x09, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x91, 0x01, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x03, 0x52, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, + 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0x53, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xab, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x5d, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0x97, 0x01, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, + 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, - 0x4b, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, - 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x91, 0x01, 0x0a, - 0x1c, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, - 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x64, 0x12, 0x50, + 0x4d, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x94, + 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x21, 0x0a, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4c, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x22, 0x8a, 0x01, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x07, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x12, 0x50, + 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x22, 0x60, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x40, 0x0a, 0x0e, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x73, 0x22, 0x8b, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x07, 0x74, 0x79, 0x70, 0x65, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x51, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, - 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x22, 0x94, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x52, 0x0a, 0x1e, 0x47, 0x65, - 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x94, - 0x01, 0x0a, 0x21, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x49, 0x64, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x22, 0x64, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x43, 0x0a, 0x0f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x22, 0xa9, 0x01, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x22, 0x55, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x0c, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x89, 0x01, 0x0a, 0x1a, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, + 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x79, 0x70, 0x65, + 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x07, 0x74, 0x79, 0x70, 0x65, + 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x56, 0x0a, 0x22, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x96, 0x01, - 0x0a, 0x23, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x49, 0x64, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x5c, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x73, 0x22, 0xa3, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x6f, 0x0a, 0x13, 0x47, 0x65, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, + 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xe9, 0x01, 0x0a, 0x18, 0x47, + 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x75, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, + 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xd5, 0x01, + 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x79, 0x54, 0x79, + 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x79, 0x70, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x51, 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, + 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x8b, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, + 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x58, 0x0a, 0x24, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, - 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, - 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, - 0x22, 0xcc, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, - 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4b, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x73, 0x22, 0x91, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x61, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x49, 0x64, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x51, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x94, 0x01, 0x0a, 0x1d, 0x47, + 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, + 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x22, 0x52, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, + 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x94, 0x01, 0x0a, 0x21, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x56, 0x0a, 0x22, + 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x73, 0x22, 0x96, 0x01, 0x0a, 0x23, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, + 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x12, 0x50, 0x0a, 0x13, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x58, 0x0a, + 0x24, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x08, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x22, 0x9b, 0x01, 0x0a, 0x22, 0x47, 0x65, 0x74, 0x50, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, + 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x03, 0x42, 0x02, 0x10, 0x01, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xe1, 0x02, 0x0a, 0x23, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, + 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x3e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, + 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x1a, 0x57, 0x0a, 0x16, 0x50, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x50, 0x65, 0x72, 0x43, 0x68, + 0x69, 0x6c, 0x64, 0x12, 0x3d, 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x52, 0x0e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x73, 0x1a, 0x84, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x5d, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x47, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x50, 0x65, 0x72, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9d, 0x01, 0x0a, 0x24, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x42, 0x02, 0x10, 0x01, 0x52, 0x0a, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xf1, 0x02, 0x0a, 0x25, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x5c, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x1a, 0x5e, 0x0a, 0x19, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x50, 0x65, 0x72, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x41, + 0x0a, 0x11, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, + 0x10, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x1a, 0x89, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x62, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, + 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x50, 0x65, 0x72, 0x50, 0x61, 0x72, 0x65, + 0x6e, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xcc, 0x01, + 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x12, 0x3b, 0x0a, + 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, + 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x7c, 0x0a, 0x1d, + 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, + 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, + 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xcd, 0x01, 0x0a, 0x1d, 0x47, + 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x12, 0x3b, 0x0a, 0x07, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xd2, 0x01, 0x0a, 0x1e, 0x47, + 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, + 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, + 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, - 0x7c, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, - 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x33, 0x0a, 0x09, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x09, 0x61, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x70, 0x61, - 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, - 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xcd, 0x01, - 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, - 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x64, 0x12, 0x3b, - 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0xab, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, + 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x07, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, + 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xd2, 0x01, - 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, - 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x36, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x65, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x65, 0x78, 0x74, - 0x5f, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x6e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x67, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, - 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0xab, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, - 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, - 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, - 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, - 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x50, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, - 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x73, - 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, - 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x08, 0x73, 0x75, 0x62, 0x67, 0x72, 0x61, - 0x70, 0x68, 0x32, 0x99, 0x22, 0x0a, 0x14, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5e, 0x0a, 0x0f, 0x50, - 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x61, 0x0a, 0x10, 0x50, - 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, - 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, - 0x0a, 0x0e, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, - 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x08, 0x50, - 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0c, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, - 0x0d, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x21, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x09, 0x50, 0x75, 0x74, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1d, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0c, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0b, - 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x8b, 0x01, 0x0a, 0x1e, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, - 0x0a, 0x11, 0x50, 0x75, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x50, 0x75, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x50, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x50, 0x0a, + 0x17, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x73, 0x75, 0x62, 0x67, + 0x72, 0x61, 0x70, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, + 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x08, 0x73, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x22, + 0x95, 0x02, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, + 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x6d, 0x0a, + 0x1e, 0x6c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, + 0x68, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, + 0x61, 0x70, 0x68, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x1b, 0x6c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x0a, 0x09, + 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x73, 0x6b, 0x52, 0x08, 0x72, 0x65, 0x61, + 0x64, 0x4d, 0x61, 0x73, 0x6b, 0x12, 0x50, 0x0a, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x62, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x4c, 0x69, + 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x10, 0x6c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, + 0x5f, 0x73, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x69, + 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x0f, 0x6c, 0x69, 0x6e, 0x65, + 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x32, 0x96, 0x2c, 0x0a, 0x14, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x5e, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x28, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x61, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x61, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x15, 0x47, 0x65, 0x74, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, - 0x49, 0x44, 0x12, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x61, 0x0a, 0x10, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x0e, 0x50, 0x75, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x08, 0x50, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x12, 0x1c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, + 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x55, 0x0a, 0x0c, 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, + 0x20, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, + 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x50, 0x75, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x0d, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x4c, 0x0a, 0x09, 0x50, 0x75, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1d, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, + 0x0a, 0x0c, 0x50, 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, + 0x75, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x67, 0x0a, 0x12, 0x50, 0x75, 0x74, 0x4c, 0x69, 0x6e, 0x65, + 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x12, 0x26, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x4c, 0x69, 0x6e, + 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x50, 0x75, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, + 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, + 0x0a, 0x0b, 0x50, 0x75, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x1f, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, + 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x8b, 0x01, 0x0a, 0x1e, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x32, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x41, 0x74, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x41, 0x6e, 0x64, 0x41, 0x73, 0x73, 0x6f, 0x63, 0x69, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x64, 0x0a, 0x11, 0x50, 0x75, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x50, 0x75, 0x74, 0x50, 0x61, + 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x28, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, + 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x61, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x61, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x24, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, - 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, - 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x11, 0x47, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, - 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x5b, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, - 0x0a, 0x13, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x42, 0x79, 0x49, 0x44, 0x12, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, + 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, + 0x11, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, + 0x65, 0x73, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, - 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, + 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x6a, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x28, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, + 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x0f, + 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, + 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0c, + 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x20, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x49, 0x44, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, + 0x0b, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x61, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, + 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, + 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x0f, 0x47, 0x65, - 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x23, 0x2e, + 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0c, 0x47, 0x65, - 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x58, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x21, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0b, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x6d, 0x6c, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x61, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, - 0x79, 0x49, 0x44, 0x12, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, - 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x64, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x12, 0x23, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x67, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x26, - 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x6a, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, - 0x11, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x79, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, - 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, - 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, - 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, - 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7c, - 0x0a, 0x19, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, - 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x6c, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, - 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x49, 0x44, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x67, 0x0a, 0x12, 0x47, 0x65, + 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, + 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, - 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x76, 0x0a, 0x17, - 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, - 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x79, 0x54, - 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x55, 0x52, 0x49, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x55, 0x52, 0x49, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x55, 0x52, 0x49, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x76, 0x0a, 0x17, 0x47, 0x65, - 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x12, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, - 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x73, 0x12, 0x2a, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, - 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x12, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x6c, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, + 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x64, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, - 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7f, - 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x2e, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x6d, - 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x85, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x12, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x12, 0x29, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, - 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x6c, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x79, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, + 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2d, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, + 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x7c, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, + 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, + 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x76, + 0x0a, 0x17, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x79, 0x54, 0x79, + 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x42, 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, + 0x79, 0x54, 0x79, 0x70, 0x65, 0x41, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x64, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x55, 0x52, 0x49, 0x12, 0x25, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, - 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x12, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x55, 0x52, 0x49, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x55, + 0x52, 0x49, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x76, 0x0a, 0x17, + 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x12, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, + 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x73, 0x12, 0x2a, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x49, 0x44, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x49, 0x44, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7c, 0x0a, 0x19, 0x47, 0x65, 0x74, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x2d, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, + 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, + 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7f, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x79, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x49, 0x64, 0x73, 0x12, 0x2c, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, + 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x88, 0x01, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x31, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8b, + 0x01, 0x0a, 0x1e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, + 0x73, 0x12, 0x32, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, + 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, + 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x85, 0x01, 0x0a, + 0x1c, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x42, 0x79, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x12, 0x30, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x31, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x54, 0x79, 0x70, 0x65, 0x73, 0x42, 0x79, 0x45, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x49, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x29, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x73, 0x42, 0x79, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, + 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7f, 0x0a, 0x1a, 0x47, + 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, + 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x85, 0x01, 0x0a, + 0x1c, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x30, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, + 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2b, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, - 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, - 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, - 0x68, 0x12, 0x23, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, - 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x31, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x82, 0x01, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, + 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x73, 0x12, 0x2f, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x88, 0x01, 0x0a, 0x1d, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x12, 0x31, 0x2e, 0x6d, 0x6c, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x69, + 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x42, 0x79, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, + 0x43, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, + 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x70, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x29, 0x2e, + 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x73, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x45, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x12, 0x2a, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, + 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x43, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x6d, + 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x0f, 0x47, + 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x12, 0x23, + 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, + 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x47, 0x72, 0x61, 0x70, + 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x67, 0x0a, 0x12, 0x47, + 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, + 0x68, 0x12, 0x26, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, + 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6d, 0x6c, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x6e, 0x65, 0x61, + 0x67, 0x65, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, } var ( @@ -6310,305 +8398,399 @@ func file_ml_metadata_proto_metadata_store_service_proto_rawDescGZIP() []byte { return file_ml_metadata_proto_metadata_store_service_proto_rawDescData } -var file_ml_metadata_proto_metadata_store_service_proto_msgTypes = make([]protoimpl.MessageInfo, 92) +var file_ml_metadata_proto_metadata_store_service_proto_msgTypes = make([]protoimpl.MessageInfo, 118) var file_ml_metadata_proto_metadata_store_service_proto_goTypes = []interface{}{ - (*ArtifactAndType)(nil), // 0: ml_metadata.ArtifactAndType - (*ArtifactStructMap)(nil), // 1: ml_metadata.ArtifactStructMap - (*ArtifactStructList)(nil), // 2: ml_metadata.ArtifactStructList - (*ArtifactStruct)(nil), // 3: ml_metadata.ArtifactStruct - (*PutArtifactsRequest)(nil), // 4: ml_metadata.PutArtifactsRequest - (*PutArtifactsResponse)(nil), // 5: ml_metadata.PutArtifactsResponse - (*PutArtifactTypeRequest)(nil), // 6: ml_metadata.PutArtifactTypeRequest - (*PutArtifactTypeResponse)(nil), // 7: ml_metadata.PutArtifactTypeResponse - (*PutExecutionsRequest)(nil), // 8: ml_metadata.PutExecutionsRequest - (*PutExecutionsResponse)(nil), // 9: ml_metadata.PutExecutionsResponse - (*PutExecutionTypeRequest)(nil), // 10: ml_metadata.PutExecutionTypeRequest - (*PutExecutionTypeResponse)(nil), // 11: ml_metadata.PutExecutionTypeResponse - (*PutEventsRequest)(nil), // 12: ml_metadata.PutEventsRequest - (*PutEventsResponse)(nil), // 13: ml_metadata.PutEventsResponse - (*PutExecutionRequest)(nil), // 14: ml_metadata.PutExecutionRequest - (*PutExecutionResponse)(nil), // 15: ml_metadata.PutExecutionResponse - (*PutTypesRequest)(nil), // 16: ml_metadata.PutTypesRequest - (*PutTypesResponse)(nil), // 17: ml_metadata.PutTypesResponse - (*PutContextTypeRequest)(nil), // 18: ml_metadata.PutContextTypeRequest - (*PutContextTypeResponse)(nil), // 19: ml_metadata.PutContextTypeResponse - (*PutContextsRequest)(nil), // 20: ml_metadata.PutContextsRequest - (*PutContextsResponse)(nil), // 21: ml_metadata.PutContextsResponse - (*PutAttributionsAndAssociationsRequest)(nil), // 22: ml_metadata.PutAttributionsAndAssociationsRequest - (*PutAttributionsAndAssociationsResponse)(nil), // 23: ml_metadata.PutAttributionsAndAssociationsResponse - (*PutParentContextsRequest)(nil), // 24: ml_metadata.PutParentContextsRequest - (*PutParentContextsResponse)(nil), // 25: ml_metadata.PutParentContextsResponse - (*GetArtifactsByTypeRequest)(nil), // 26: ml_metadata.GetArtifactsByTypeRequest - (*GetArtifactsByTypeResponse)(nil), // 27: ml_metadata.GetArtifactsByTypeResponse - (*GetArtifactByTypeAndNameRequest)(nil), // 28: ml_metadata.GetArtifactByTypeAndNameRequest - (*GetArtifactByTypeAndNameResponse)(nil), // 29: ml_metadata.GetArtifactByTypeAndNameResponse - (*GetArtifactsByIDRequest)(nil), // 30: ml_metadata.GetArtifactsByIDRequest - (*GetArtifactsByIDResponse)(nil), // 31: ml_metadata.GetArtifactsByIDResponse - (*GetArtifactsRequest)(nil), // 32: ml_metadata.GetArtifactsRequest - (*GetArtifactsResponse)(nil), // 33: ml_metadata.GetArtifactsResponse - (*GetArtifactsByURIRequest)(nil), // 34: ml_metadata.GetArtifactsByURIRequest - (*GetArtifactsByURIResponse)(nil), // 35: ml_metadata.GetArtifactsByURIResponse - (*GetExecutionsRequest)(nil), // 36: ml_metadata.GetExecutionsRequest - (*GetExecutionsResponse)(nil), // 37: ml_metadata.GetExecutionsResponse - (*GetArtifactTypeRequest)(nil), // 38: ml_metadata.GetArtifactTypeRequest - (*GetArtifactTypeResponse)(nil), // 39: ml_metadata.GetArtifactTypeResponse - (*GetArtifactTypesRequest)(nil), // 40: ml_metadata.GetArtifactTypesRequest - (*GetArtifactTypesResponse)(nil), // 41: ml_metadata.GetArtifactTypesResponse - (*GetExecutionTypesRequest)(nil), // 42: ml_metadata.GetExecutionTypesRequest - (*GetExecutionTypesResponse)(nil), // 43: ml_metadata.GetExecutionTypesResponse - (*GetContextTypesRequest)(nil), // 44: ml_metadata.GetContextTypesRequest - (*GetContextTypesResponse)(nil), // 45: ml_metadata.GetContextTypesResponse - (*GetExecutionsByTypeRequest)(nil), // 46: ml_metadata.GetExecutionsByTypeRequest - (*GetExecutionsByTypeResponse)(nil), // 47: ml_metadata.GetExecutionsByTypeResponse - (*GetExecutionByTypeAndNameRequest)(nil), // 48: ml_metadata.GetExecutionByTypeAndNameRequest - (*GetExecutionByTypeAndNameResponse)(nil), // 49: ml_metadata.GetExecutionByTypeAndNameResponse - (*GetExecutionsByIDRequest)(nil), // 50: ml_metadata.GetExecutionsByIDRequest - (*GetExecutionsByIDResponse)(nil), // 51: ml_metadata.GetExecutionsByIDResponse - (*GetExecutionTypeRequest)(nil), // 52: ml_metadata.GetExecutionTypeRequest - (*GetExecutionTypeResponse)(nil), // 53: ml_metadata.GetExecutionTypeResponse - (*GetEventsByExecutionIDsRequest)(nil), // 54: ml_metadata.GetEventsByExecutionIDsRequest - (*GetEventsByExecutionIDsResponse)(nil), // 55: ml_metadata.GetEventsByExecutionIDsResponse - (*GetEventsByArtifactIDsRequest)(nil), // 56: ml_metadata.GetEventsByArtifactIDsRequest - (*GetEventsByArtifactIDsResponse)(nil), // 57: ml_metadata.GetEventsByArtifactIDsResponse - (*GetArtifactTypesByIDRequest)(nil), // 58: ml_metadata.GetArtifactTypesByIDRequest - (*GetArtifactTypesByIDResponse)(nil), // 59: ml_metadata.GetArtifactTypesByIDResponse - (*GetExecutionTypesByIDRequest)(nil), // 60: ml_metadata.GetExecutionTypesByIDRequest - (*GetExecutionTypesByIDResponse)(nil), // 61: ml_metadata.GetExecutionTypesByIDResponse - (*GetContextTypeRequest)(nil), // 62: ml_metadata.GetContextTypeRequest - (*GetContextTypeResponse)(nil), // 63: ml_metadata.GetContextTypeResponse - (*GetContextTypesByIDRequest)(nil), // 64: ml_metadata.GetContextTypesByIDRequest - (*GetContextTypesByIDResponse)(nil), // 65: ml_metadata.GetContextTypesByIDResponse - (*GetContextsRequest)(nil), // 66: ml_metadata.GetContextsRequest - (*GetContextsResponse)(nil), // 67: ml_metadata.GetContextsResponse - (*GetContextsByTypeRequest)(nil), // 68: ml_metadata.GetContextsByTypeRequest - (*GetContextsByTypeResponse)(nil), // 69: ml_metadata.GetContextsByTypeResponse - (*GetContextByTypeAndNameRequest)(nil), // 70: ml_metadata.GetContextByTypeAndNameRequest - (*GetContextByTypeAndNameResponse)(nil), // 71: ml_metadata.GetContextByTypeAndNameResponse - (*GetContextsByIDRequest)(nil), // 72: ml_metadata.GetContextsByIDRequest - (*GetContextsByIDResponse)(nil), // 73: ml_metadata.GetContextsByIDResponse - (*GetContextsByArtifactRequest)(nil), // 74: ml_metadata.GetContextsByArtifactRequest - (*GetContextsByArtifactResponse)(nil), // 75: ml_metadata.GetContextsByArtifactResponse - (*GetContextsByExecutionRequest)(nil), // 76: ml_metadata.GetContextsByExecutionRequest - (*GetContextsByExecutionResponse)(nil), // 77: ml_metadata.GetContextsByExecutionResponse - (*GetParentContextsByContextRequest)(nil), // 78: ml_metadata.GetParentContextsByContextRequest - (*GetParentContextsByContextResponse)(nil), // 79: ml_metadata.GetParentContextsByContextResponse - (*GetChildrenContextsByContextRequest)(nil), // 80: ml_metadata.GetChildrenContextsByContextRequest - (*GetChildrenContextsByContextResponse)(nil), // 81: ml_metadata.GetChildrenContextsByContextResponse - (*GetArtifactsByContextRequest)(nil), // 82: ml_metadata.GetArtifactsByContextRequest - (*GetArtifactsByContextResponse)(nil), // 83: ml_metadata.GetArtifactsByContextResponse - (*GetExecutionsByContextRequest)(nil), // 84: ml_metadata.GetExecutionsByContextRequest - (*GetExecutionsByContextResponse)(nil), // 85: ml_metadata.GetExecutionsByContextResponse - (*GetLineageGraphRequest)(nil), // 86: ml_metadata.GetLineageGraphRequest - (*GetLineageGraphResponse)(nil), // 87: ml_metadata.GetLineageGraphResponse - nil, // 88: ml_metadata.ArtifactStructMap.PropertiesEntry - (*PutArtifactsRequest_Options)(nil), // 89: ml_metadata.PutArtifactsRequest.Options - (*PutExecutionRequest_ArtifactAndEvent)(nil), // 90: ml_metadata.PutExecutionRequest.ArtifactAndEvent - (*PutExecutionRequest_Options)(nil), // 91: ml_metadata.PutExecutionRequest.Options - (*Artifact)(nil), // 92: ml_metadata.Artifact - (*ArtifactType)(nil), // 93: ml_metadata.ArtifactType - (*Execution)(nil), // 94: ml_metadata.Execution - (*ExecutionType)(nil), // 95: ml_metadata.ExecutionType - (*Event)(nil), // 96: ml_metadata.Event - (*Context)(nil), // 97: ml_metadata.Context - (*ContextType)(nil), // 98: ml_metadata.ContextType - (*Attribution)(nil), // 99: ml_metadata.Attribution - (*Association)(nil), // 100: ml_metadata.Association - (*ParentContext)(nil), // 101: ml_metadata.ParentContext - (*ListOperationOptions)(nil), // 102: ml_metadata.ListOperationOptions - (*TransactionOptions)(nil), // 103: ml_metadata.TransactionOptions - (*LineageGraphQueryOptions)(nil), // 104: ml_metadata.LineageGraphQueryOptions - (*LineageGraph)(nil), // 105: ml_metadata.LineageGraph + (*ArtifactAndType)(nil), // 0: ml_metadata.ArtifactAndType + (*ArtifactStructMap)(nil), // 1: ml_metadata.ArtifactStructMap + (*ArtifactStructList)(nil), // 2: ml_metadata.ArtifactStructList + (*ArtifactStruct)(nil), // 3: ml_metadata.ArtifactStruct + (*PutArtifactsRequest)(nil), // 4: ml_metadata.PutArtifactsRequest + (*PutArtifactsResponse)(nil), // 5: ml_metadata.PutArtifactsResponse + (*PutArtifactTypeRequest)(nil), // 6: ml_metadata.PutArtifactTypeRequest + (*PutArtifactTypeResponse)(nil), // 7: ml_metadata.PutArtifactTypeResponse + (*PutExecutionsRequest)(nil), // 8: ml_metadata.PutExecutionsRequest + (*PutExecutionsResponse)(nil), // 9: ml_metadata.PutExecutionsResponse + (*PutExecutionTypeRequest)(nil), // 10: ml_metadata.PutExecutionTypeRequest + (*PutExecutionTypeResponse)(nil), // 11: ml_metadata.PutExecutionTypeResponse + (*PutEventsRequest)(nil), // 12: ml_metadata.PutEventsRequest + (*PutEventsResponse)(nil), // 13: ml_metadata.PutEventsResponse + (*PutExecutionRequest)(nil), // 14: ml_metadata.PutExecutionRequest + (*PutExecutionResponse)(nil), // 15: ml_metadata.PutExecutionResponse + (*PutLineageSubgraphRequest)(nil), // 16: ml_metadata.PutLineageSubgraphRequest + (*PutLineageSubgraphResponse)(nil), // 17: ml_metadata.PutLineageSubgraphResponse + (*PutTypesRequest)(nil), // 18: ml_metadata.PutTypesRequest + (*PutTypesResponse)(nil), // 19: ml_metadata.PutTypesResponse + (*PutContextTypeRequest)(nil), // 20: ml_metadata.PutContextTypeRequest + (*PutContextTypeResponse)(nil), // 21: ml_metadata.PutContextTypeResponse + (*PutContextsRequest)(nil), // 22: ml_metadata.PutContextsRequest + (*PutContextsResponse)(nil), // 23: ml_metadata.PutContextsResponse + (*PutAttributionsAndAssociationsRequest)(nil), // 24: ml_metadata.PutAttributionsAndAssociationsRequest + (*PutAttributionsAndAssociationsResponse)(nil), // 25: ml_metadata.PutAttributionsAndAssociationsResponse + (*PutParentContextsRequest)(nil), // 26: ml_metadata.PutParentContextsRequest + (*PutParentContextsResponse)(nil), // 27: ml_metadata.PutParentContextsResponse + (*GetArtifactsByTypeRequest)(nil), // 28: ml_metadata.GetArtifactsByTypeRequest + (*GetArtifactsByTypeResponse)(nil), // 29: ml_metadata.GetArtifactsByTypeResponse + (*GetArtifactByTypeAndNameRequest)(nil), // 30: ml_metadata.GetArtifactByTypeAndNameRequest + (*GetArtifactByTypeAndNameResponse)(nil), // 31: ml_metadata.GetArtifactByTypeAndNameResponse + (*GetArtifactsByIDRequest)(nil), // 32: ml_metadata.GetArtifactsByIDRequest + (*GetArtifactsByIDResponse)(nil), // 33: ml_metadata.GetArtifactsByIDResponse + (*GetArtifactsRequest)(nil), // 34: ml_metadata.GetArtifactsRequest + (*GetArtifactsResponse)(nil), // 35: ml_metadata.GetArtifactsResponse + (*GetArtifactsByURIRequest)(nil), // 36: ml_metadata.GetArtifactsByURIRequest + (*GetArtifactsByURIResponse)(nil), // 37: ml_metadata.GetArtifactsByURIResponse + (*GetExecutionsRequest)(nil), // 38: ml_metadata.GetExecutionsRequest + (*GetExecutionsResponse)(nil), // 39: ml_metadata.GetExecutionsResponse + (*GetArtifactTypeRequest)(nil), // 40: ml_metadata.GetArtifactTypeRequest + (*GetArtifactTypeResponse)(nil), // 41: ml_metadata.GetArtifactTypeResponse + (*GetArtifactTypesRequest)(nil), // 42: ml_metadata.GetArtifactTypesRequest + (*GetArtifactTypesResponse)(nil), // 43: ml_metadata.GetArtifactTypesResponse + (*GetExecutionTypesRequest)(nil), // 44: ml_metadata.GetExecutionTypesRequest + (*GetExecutionTypesResponse)(nil), // 45: ml_metadata.GetExecutionTypesResponse + (*GetContextTypesRequest)(nil), // 46: ml_metadata.GetContextTypesRequest + (*GetContextTypesResponse)(nil), // 47: ml_metadata.GetContextTypesResponse + (*GetArtifactsByExternalIdsRequest)(nil), // 48: ml_metadata.GetArtifactsByExternalIdsRequest + (*GetArtifactsByExternalIdsResponse)(nil), // 49: ml_metadata.GetArtifactsByExternalIdsResponse + (*GetExecutionsByExternalIdsRequest)(nil), // 50: ml_metadata.GetExecutionsByExternalIdsRequest + (*GetExecutionsByExternalIdsResponse)(nil), // 51: ml_metadata.GetExecutionsByExternalIdsResponse + (*GetContextsByExternalIdsRequest)(nil), // 52: ml_metadata.GetContextsByExternalIdsRequest + (*GetContextsByExternalIdsResponse)(nil), // 53: ml_metadata.GetContextsByExternalIdsResponse + (*GetArtifactTypesByExternalIdsRequest)(nil), // 54: ml_metadata.GetArtifactTypesByExternalIdsRequest + (*GetArtifactTypesByExternalIdsResponse)(nil), // 55: ml_metadata.GetArtifactTypesByExternalIdsResponse + (*GetExecutionTypesByExternalIdsRequest)(nil), // 56: ml_metadata.GetExecutionTypesByExternalIdsRequest + (*GetExecutionTypesByExternalIdsResponse)(nil), // 57: ml_metadata.GetExecutionTypesByExternalIdsResponse + (*GetContextTypesByExternalIdsRequest)(nil), // 58: ml_metadata.GetContextTypesByExternalIdsRequest + (*GetContextTypesByExternalIdsResponse)(nil), // 59: ml_metadata.GetContextTypesByExternalIdsResponse + (*GetExecutionsByTypeRequest)(nil), // 60: ml_metadata.GetExecutionsByTypeRequest + (*GetExecutionsByTypeResponse)(nil), // 61: ml_metadata.GetExecutionsByTypeResponse + (*GetExecutionByTypeAndNameRequest)(nil), // 62: ml_metadata.GetExecutionByTypeAndNameRequest + (*GetExecutionByTypeAndNameResponse)(nil), // 63: ml_metadata.GetExecutionByTypeAndNameResponse + (*GetExecutionsByIDRequest)(nil), // 64: ml_metadata.GetExecutionsByIDRequest + (*GetExecutionsByIDResponse)(nil), // 65: ml_metadata.GetExecutionsByIDResponse + (*GetExecutionTypeRequest)(nil), // 66: ml_metadata.GetExecutionTypeRequest + (*GetExecutionTypeResponse)(nil), // 67: ml_metadata.GetExecutionTypeResponse + (*GetEventsByExecutionIDsRequest)(nil), // 68: ml_metadata.GetEventsByExecutionIDsRequest + (*GetEventsByExecutionIDsResponse)(nil), // 69: ml_metadata.GetEventsByExecutionIDsResponse + (*GetEventsByArtifactIDsRequest)(nil), // 70: ml_metadata.GetEventsByArtifactIDsRequest + (*GetEventsByArtifactIDsResponse)(nil), // 71: ml_metadata.GetEventsByArtifactIDsResponse + (*GetArtifactTypesByIDRequest)(nil), // 72: ml_metadata.GetArtifactTypesByIDRequest + (*GetArtifactTypesByIDResponse)(nil), // 73: ml_metadata.GetArtifactTypesByIDResponse + (*GetExecutionTypesByIDRequest)(nil), // 74: ml_metadata.GetExecutionTypesByIDRequest + (*GetExecutionTypesByIDResponse)(nil), // 75: ml_metadata.GetExecutionTypesByIDResponse + (*GetContextTypeRequest)(nil), // 76: ml_metadata.GetContextTypeRequest + (*GetContextTypeResponse)(nil), // 77: ml_metadata.GetContextTypeResponse + (*GetContextTypesByIDRequest)(nil), // 78: ml_metadata.GetContextTypesByIDRequest + (*GetContextTypesByIDResponse)(nil), // 79: ml_metadata.GetContextTypesByIDResponse + (*GetContextsRequest)(nil), // 80: ml_metadata.GetContextsRequest + (*GetContextsResponse)(nil), // 81: ml_metadata.GetContextsResponse + (*GetContextsByTypeRequest)(nil), // 82: ml_metadata.GetContextsByTypeRequest + (*GetContextsByTypeResponse)(nil), // 83: ml_metadata.GetContextsByTypeResponse + (*GetContextByTypeAndNameRequest)(nil), // 84: ml_metadata.GetContextByTypeAndNameRequest + (*GetContextByTypeAndNameResponse)(nil), // 85: ml_metadata.GetContextByTypeAndNameResponse + (*GetContextsByIDRequest)(nil), // 86: ml_metadata.GetContextsByIDRequest + (*GetContextsByIDResponse)(nil), // 87: ml_metadata.GetContextsByIDResponse + (*GetContextsByArtifactRequest)(nil), // 88: ml_metadata.GetContextsByArtifactRequest + (*GetContextsByArtifactResponse)(nil), // 89: ml_metadata.GetContextsByArtifactResponse + (*GetContextsByExecutionRequest)(nil), // 90: ml_metadata.GetContextsByExecutionRequest + (*GetContextsByExecutionResponse)(nil), // 91: ml_metadata.GetContextsByExecutionResponse + (*GetParentContextsByContextRequest)(nil), // 92: ml_metadata.GetParentContextsByContextRequest + (*GetParentContextsByContextResponse)(nil), // 93: ml_metadata.GetParentContextsByContextResponse + (*GetChildrenContextsByContextRequest)(nil), // 94: ml_metadata.GetChildrenContextsByContextRequest + (*GetChildrenContextsByContextResponse)(nil), // 95: ml_metadata.GetChildrenContextsByContextResponse + (*GetParentContextsByContextsRequest)(nil), // 96: ml_metadata.GetParentContextsByContextsRequest + (*GetParentContextsByContextsResponse)(nil), // 97: ml_metadata.GetParentContextsByContextsResponse + (*GetChildrenContextsByContextsRequest)(nil), // 98: ml_metadata.GetChildrenContextsByContextsRequest + (*GetChildrenContextsByContextsResponse)(nil), // 99: ml_metadata.GetChildrenContextsByContextsResponse + (*GetArtifactsByContextRequest)(nil), // 100: ml_metadata.GetArtifactsByContextRequest + (*GetArtifactsByContextResponse)(nil), // 101: ml_metadata.GetArtifactsByContextResponse + (*GetExecutionsByContextRequest)(nil), // 102: ml_metadata.GetExecutionsByContextRequest + (*GetExecutionsByContextResponse)(nil), // 103: ml_metadata.GetExecutionsByContextResponse + (*GetLineageGraphRequest)(nil), // 104: ml_metadata.GetLineageGraphRequest + (*GetLineageGraphResponse)(nil), // 105: ml_metadata.GetLineageGraphResponse + (*GetLineageSubgraphRequest)(nil), // 106: ml_metadata.GetLineageSubgraphRequest + (*GetLineageSubgraphResponse)(nil), // 107: ml_metadata.GetLineageSubgraphResponse + nil, // 108: ml_metadata.ArtifactStructMap.PropertiesEntry + (*PutArtifactsRequest_Options)(nil), // 109: ml_metadata.PutArtifactsRequest.Options + (*PutExecutionRequest_ArtifactAndEvent)(nil), // 110: ml_metadata.PutExecutionRequest.ArtifactAndEvent + (*PutExecutionRequest_Options)(nil), // 111: ml_metadata.PutExecutionRequest.Options + (*PutLineageSubgraphRequest_EventEdge)(nil), // 112: ml_metadata.PutLineageSubgraphRequest.EventEdge + (*PutLineageSubgraphRequest_Options)(nil), // 113: ml_metadata.PutLineageSubgraphRequest.Options + (*GetParentContextsByContextsResponse_ParentContextsPerChild)(nil), // 114: ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild + nil, // 115: ml_metadata.GetParentContextsByContextsResponse.ContextsEntry + (*GetChildrenContextsByContextsResponse_ChildrenContextsPerParent)(nil), // 116: ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent + nil, // 117: ml_metadata.GetChildrenContextsByContextsResponse.ContextsEntry + (*Artifact)(nil), // 118: ml_metadata.Artifact + (*ArtifactType)(nil), // 119: ml_metadata.ArtifactType + (*TransactionOptions)(nil), // 120: ml_metadata.TransactionOptions + (*fieldmaskpb.FieldMask)(nil), // 121: google.protobuf.FieldMask + (*Execution)(nil), // 122: ml_metadata.Execution + (*ExecutionType)(nil), // 123: ml_metadata.ExecutionType + (*Event)(nil), // 124: ml_metadata.Event + (*Context)(nil), // 125: ml_metadata.Context + (*ContextType)(nil), // 126: ml_metadata.ContextType + (*Attribution)(nil), // 127: ml_metadata.Attribution + (*Association)(nil), // 128: ml_metadata.Association + (*ParentContext)(nil), // 129: ml_metadata.ParentContext + (*ListOperationOptions)(nil), // 130: ml_metadata.ListOperationOptions + (*LineageGraphQueryOptions)(nil), // 131: ml_metadata.LineageGraphQueryOptions + (*LineageGraph)(nil), // 132: ml_metadata.LineageGraph + (*LineageSubgraphQueryOptions)(nil), // 133: ml_metadata.LineageSubgraphQueryOptions } var file_ml_metadata_proto_metadata_store_service_proto_depIdxs = []int32{ - 92, // 0: ml_metadata.ArtifactAndType.artifact:type_name -> ml_metadata.Artifact - 93, // 1: ml_metadata.ArtifactAndType.type:type_name -> ml_metadata.ArtifactType - 88, // 2: ml_metadata.ArtifactStructMap.properties:type_name -> ml_metadata.ArtifactStructMap.PropertiesEntry + 118, // 0: ml_metadata.ArtifactAndType.artifact:type_name -> ml_metadata.Artifact + 119, // 1: ml_metadata.ArtifactAndType.type:type_name -> ml_metadata.ArtifactType + 108, // 2: ml_metadata.ArtifactStructMap.properties:type_name -> ml_metadata.ArtifactStructMap.PropertiesEntry 3, // 3: ml_metadata.ArtifactStructList.elements:type_name -> ml_metadata.ArtifactStruct 0, // 4: ml_metadata.ArtifactStruct.artifact:type_name -> ml_metadata.ArtifactAndType 1, // 5: ml_metadata.ArtifactStruct.map:type_name -> ml_metadata.ArtifactStructMap 2, // 6: ml_metadata.ArtifactStruct.list:type_name -> ml_metadata.ArtifactStructList - 92, // 7: ml_metadata.PutArtifactsRequest.artifacts:type_name -> ml_metadata.Artifact - 89, // 8: ml_metadata.PutArtifactsRequest.options:type_name -> ml_metadata.PutArtifactsRequest.Options - 93, // 9: ml_metadata.PutArtifactTypeRequest.artifact_type:type_name -> ml_metadata.ArtifactType - 94, // 10: ml_metadata.PutExecutionsRequest.executions:type_name -> ml_metadata.Execution - 95, // 11: ml_metadata.PutExecutionTypeRequest.execution_type:type_name -> ml_metadata.ExecutionType - 96, // 12: ml_metadata.PutEventsRequest.events:type_name -> ml_metadata.Event - 94, // 13: ml_metadata.PutExecutionRequest.execution:type_name -> ml_metadata.Execution - 90, // 14: ml_metadata.PutExecutionRequest.artifact_event_pairs:type_name -> ml_metadata.PutExecutionRequest.ArtifactAndEvent - 97, // 15: ml_metadata.PutExecutionRequest.contexts:type_name -> ml_metadata.Context - 91, // 16: ml_metadata.PutExecutionRequest.options:type_name -> ml_metadata.PutExecutionRequest.Options - 93, // 17: ml_metadata.PutTypesRequest.artifact_types:type_name -> ml_metadata.ArtifactType - 95, // 18: ml_metadata.PutTypesRequest.execution_types:type_name -> ml_metadata.ExecutionType - 98, // 19: ml_metadata.PutTypesRequest.context_types:type_name -> ml_metadata.ContextType - 98, // 20: ml_metadata.PutContextTypeRequest.context_type:type_name -> ml_metadata.ContextType - 97, // 21: ml_metadata.PutContextsRequest.contexts:type_name -> ml_metadata.Context - 99, // 22: ml_metadata.PutAttributionsAndAssociationsRequest.attributions:type_name -> ml_metadata.Attribution - 100, // 23: ml_metadata.PutAttributionsAndAssociationsRequest.associations:type_name -> ml_metadata.Association - 101, // 24: ml_metadata.PutParentContextsRequest.parent_contexts:type_name -> ml_metadata.ParentContext - 102, // 25: ml_metadata.GetArtifactsByTypeRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 26: ml_metadata.GetArtifactsByTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 92, // 27: ml_metadata.GetArtifactsByTypeResponse.artifacts:type_name -> ml_metadata.Artifact - 103, // 28: ml_metadata.GetArtifactByTypeAndNameRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 92, // 29: ml_metadata.GetArtifactByTypeAndNameResponse.artifact:type_name -> ml_metadata.Artifact - 103, // 30: ml_metadata.GetArtifactsByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 92, // 31: ml_metadata.GetArtifactsByIDResponse.artifacts:type_name -> ml_metadata.Artifact - 102, // 32: ml_metadata.GetArtifactsRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 33: ml_metadata.GetArtifactsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 92, // 34: ml_metadata.GetArtifactsResponse.artifacts:type_name -> ml_metadata.Artifact - 103, // 35: ml_metadata.GetArtifactsByURIRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 92, // 36: ml_metadata.GetArtifactsByURIResponse.artifacts:type_name -> ml_metadata.Artifact - 102, // 37: ml_metadata.GetExecutionsRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 38: ml_metadata.GetExecutionsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 94, // 39: ml_metadata.GetExecutionsResponse.executions:type_name -> ml_metadata.Execution - 103, // 40: ml_metadata.GetArtifactTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 93, // 41: ml_metadata.GetArtifactTypeResponse.artifact_type:type_name -> ml_metadata.ArtifactType - 103, // 42: ml_metadata.GetArtifactTypesRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 93, // 43: ml_metadata.GetArtifactTypesResponse.artifact_types:type_name -> ml_metadata.ArtifactType - 103, // 44: ml_metadata.GetExecutionTypesRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 95, // 45: ml_metadata.GetExecutionTypesResponse.execution_types:type_name -> ml_metadata.ExecutionType - 103, // 46: ml_metadata.GetContextTypesRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 98, // 47: ml_metadata.GetContextTypesResponse.context_types:type_name -> ml_metadata.ContextType - 102, // 48: ml_metadata.GetExecutionsByTypeRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 49: ml_metadata.GetExecutionsByTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 94, // 50: ml_metadata.GetExecutionsByTypeResponse.executions:type_name -> ml_metadata.Execution - 103, // 51: ml_metadata.GetExecutionByTypeAndNameRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 94, // 52: ml_metadata.GetExecutionByTypeAndNameResponse.execution:type_name -> ml_metadata.Execution - 103, // 53: ml_metadata.GetExecutionsByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 94, // 54: ml_metadata.GetExecutionsByIDResponse.executions:type_name -> ml_metadata.Execution - 103, // 55: ml_metadata.GetExecutionTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 95, // 56: ml_metadata.GetExecutionTypeResponse.execution_type:type_name -> ml_metadata.ExecutionType - 103, // 57: ml_metadata.GetEventsByExecutionIDsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 96, // 58: ml_metadata.GetEventsByExecutionIDsResponse.events:type_name -> ml_metadata.Event - 103, // 59: ml_metadata.GetEventsByArtifactIDsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 96, // 60: ml_metadata.GetEventsByArtifactIDsResponse.events:type_name -> ml_metadata.Event - 103, // 61: ml_metadata.GetArtifactTypesByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 93, // 62: ml_metadata.GetArtifactTypesByIDResponse.artifact_types:type_name -> ml_metadata.ArtifactType - 103, // 63: ml_metadata.GetExecutionTypesByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 95, // 64: ml_metadata.GetExecutionTypesByIDResponse.execution_types:type_name -> ml_metadata.ExecutionType - 103, // 65: ml_metadata.GetContextTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 98, // 66: ml_metadata.GetContextTypeResponse.context_type:type_name -> ml_metadata.ContextType - 103, // 67: ml_metadata.GetContextTypesByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 98, // 68: ml_metadata.GetContextTypesByIDResponse.context_types:type_name -> ml_metadata.ContextType - 102, // 69: ml_metadata.GetContextsRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 70: ml_metadata.GetContextsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 71: ml_metadata.GetContextsResponse.contexts:type_name -> ml_metadata.Context - 102, // 72: ml_metadata.GetContextsByTypeRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 73: ml_metadata.GetContextsByTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 74: ml_metadata.GetContextsByTypeResponse.contexts:type_name -> ml_metadata.Context - 103, // 75: ml_metadata.GetContextByTypeAndNameRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 76: ml_metadata.GetContextByTypeAndNameResponse.context:type_name -> ml_metadata.Context - 103, // 77: ml_metadata.GetContextsByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 78: ml_metadata.GetContextsByIDResponse.contexts:type_name -> ml_metadata.Context - 103, // 79: ml_metadata.GetContextsByArtifactRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 80: ml_metadata.GetContextsByArtifactResponse.contexts:type_name -> ml_metadata.Context - 103, // 81: ml_metadata.GetContextsByExecutionRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 82: ml_metadata.GetContextsByExecutionResponse.contexts:type_name -> ml_metadata.Context - 103, // 83: ml_metadata.GetParentContextsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 84: ml_metadata.GetParentContextsByContextResponse.contexts:type_name -> ml_metadata.Context - 103, // 85: ml_metadata.GetChildrenContextsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 97, // 86: ml_metadata.GetChildrenContextsByContextResponse.contexts:type_name -> ml_metadata.Context - 102, // 87: ml_metadata.GetArtifactsByContextRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 88: ml_metadata.GetArtifactsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 92, // 89: ml_metadata.GetArtifactsByContextResponse.artifacts:type_name -> ml_metadata.Artifact - 102, // 90: ml_metadata.GetExecutionsByContextRequest.options:type_name -> ml_metadata.ListOperationOptions - 103, // 91: ml_metadata.GetExecutionsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 94, // 92: ml_metadata.GetExecutionsByContextResponse.executions:type_name -> ml_metadata.Execution - 103, // 93: ml_metadata.GetExecutionsByContextResponse.transaction_options:type_name -> ml_metadata.TransactionOptions - 104, // 94: ml_metadata.GetLineageGraphRequest.options:type_name -> ml_metadata.LineageGraphQueryOptions - 103, // 95: ml_metadata.GetLineageGraphRequest.transaction_options:type_name -> ml_metadata.TransactionOptions - 105, // 96: ml_metadata.GetLineageGraphResponse.subgraph:type_name -> ml_metadata.LineageGraph - 3, // 97: ml_metadata.ArtifactStructMap.PropertiesEntry.value:type_name -> ml_metadata.ArtifactStruct - 92, // 98: ml_metadata.PutExecutionRequest.ArtifactAndEvent.artifact:type_name -> ml_metadata.Artifact - 96, // 99: ml_metadata.PutExecutionRequest.ArtifactAndEvent.event:type_name -> ml_metadata.Event - 6, // 100: ml_metadata.MetadataStoreService.PutArtifactType:input_type -> ml_metadata.PutArtifactTypeRequest - 10, // 101: ml_metadata.MetadataStoreService.PutExecutionType:input_type -> ml_metadata.PutExecutionTypeRequest - 18, // 102: ml_metadata.MetadataStoreService.PutContextType:input_type -> ml_metadata.PutContextTypeRequest - 16, // 103: ml_metadata.MetadataStoreService.PutTypes:input_type -> ml_metadata.PutTypesRequest - 4, // 104: ml_metadata.MetadataStoreService.PutArtifacts:input_type -> ml_metadata.PutArtifactsRequest - 8, // 105: ml_metadata.MetadataStoreService.PutExecutions:input_type -> ml_metadata.PutExecutionsRequest - 12, // 106: ml_metadata.MetadataStoreService.PutEvents:input_type -> ml_metadata.PutEventsRequest - 14, // 107: ml_metadata.MetadataStoreService.PutExecution:input_type -> ml_metadata.PutExecutionRequest - 20, // 108: ml_metadata.MetadataStoreService.PutContexts:input_type -> ml_metadata.PutContextsRequest - 22, // 109: ml_metadata.MetadataStoreService.PutAttributionsAndAssociations:input_type -> ml_metadata.PutAttributionsAndAssociationsRequest - 24, // 110: ml_metadata.MetadataStoreService.PutParentContexts:input_type -> ml_metadata.PutParentContextsRequest - 38, // 111: ml_metadata.MetadataStoreService.GetArtifactType:input_type -> ml_metadata.GetArtifactTypeRequest - 58, // 112: ml_metadata.MetadataStoreService.GetArtifactTypesByID:input_type -> ml_metadata.GetArtifactTypesByIDRequest - 40, // 113: ml_metadata.MetadataStoreService.GetArtifactTypes:input_type -> ml_metadata.GetArtifactTypesRequest - 52, // 114: ml_metadata.MetadataStoreService.GetExecutionType:input_type -> ml_metadata.GetExecutionTypeRequest - 60, // 115: ml_metadata.MetadataStoreService.GetExecutionTypesByID:input_type -> ml_metadata.GetExecutionTypesByIDRequest - 42, // 116: ml_metadata.MetadataStoreService.GetExecutionTypes:input_type -> ml_metadata.GetExecutionTypesRequest - 62, // 117: ml_metadata.MetadataStoreService.GetContextType:input_type -> ml_metadata.GetContextTypeRequest - 64, // 118: ml_metadata.MetadataStoreService.GetContextTypesByID:input_type -> ml_metadata.GetContextTypesByIDRequest - 44, // 119: ml_metadata.MetadataStoreService.GetContextTypes:input_type -> ml_metadata.GetContextTypesRequest - 32, // 120: ml_metadata.MetadataStoreService.GetArtifacts:input_type -> ml_metadata.GetArtifactsRequest - 36, // 121: ml_metadata.MetadataStoreService.GetExecutions:input_type -> ml_metadata.GetExecutionsRequest - 66, // 122: ml_metadata.MetadataStoreService.GetContexts:input_type -> ml_metadata.GetContextsRequest - 30, // 123: ml_metadata.MetadataStoreService.GetArtifactsByID:input_type -> ml_metadata.GetArtifactsByIDRequest - 50, // 124: ml_metadata.MetadataStoreService.GetExecutionsByID:input_type -> ml_metadata.GetExecutionsByIDRequest - 72, // 125: ml_metadata.MetadataStoreService.GetContextsByID:input_type -> ml_metadata.GetContextsByIDRequest - 26, // 126: ml_metadata.MetadataStoreService.GetArtifactsByType:input_type -> ml_metadata.GetArtifactsByTypeRequest - 46, // 127: ml_metadata.MetadataStoreService.GetExecutionsByType:input_type -> ml_metadata.GetExecutionsByTypeRequest - 68, // 128: ml_metadata.MetadataStoreService.GetContextsByType:input_type -> ml_metadata.GetContextsByTypeRequest - 28, // 129: ml_metadata.MetadataStoreService.GetArtifactByTypeAndName:input_type -> ml_metadata.GetArtifactByTypeAndNameRequest - 48, // 130: ml_metadata.MetadataStoreService.GetExecutionByTypeAndName:input_type -> ml_metadata.GetExecutionByTypeAndNameRequest - 70, // 131: ml_metadata.MetadataStoreService.GetContextByTypeAndName:input_type -> ml_metadata.GetContextByTypeAndNameRequest - 34, // 132: ml_metadata.MetadataStoreService.GetArtifactsByURI:input_type -> ml_metadata.GetArtifactsByURIRequest - 54, // 133: ml_metadata.MetadataStoreService.GetEventsByExecutionIDs:input_type -> ml_metadata.GetEventsByExecutionIDsRequest - 56, // 134: ml_metadata.MetadataStoreService.GetEventsByArtifactIDs:input_type -> ml_metadata.GetEventsByArtifactIDsRequest - 74, // 135: ml_metadata.MetadataStoreService.GetContextsByArtifact:input_type -> ml_metadata.GetContextsByArtifactRequest - 76, // 136: ml_metadata.MetadataStoreService.GetContextsByExecution:input_type -> ml_metadata.GetContextsByExecutionRequest - 78, // 137: ml_metadata.MetadataStoreService.GetParentContextsByContext:input_type -> ml_metadata.GetParentContextsByContextRequest - 80, // 138: ml_metadata.MetadataStoreService.GetChildrenContextsByContext:input_type -> ml_metadata.GetChildrenContextsByContextRequest - 82, // 139: ml_metadata.MetadataStoreService.GetArtifactsByContext:input_type -> ml_metadata.GetArtifactsByContextRequest - 84, // 140: ml_metadata.MetadataStoreService.GetExecutionsByContext:input_type -> ml_metadata.GetExecutionsByContextRequest - 86, // 141: ml_metadata.MetadataStoreService.GetLineageGraph:input_type -> ml_metadata.GetLineageGraphRequest - 7, // 142: ml_metadata.MetadataStoreService.PutArtifactType:output_type -> ml_metadata.PutArtifactTypeResponse - 11, // 143: ml_metadata.MetadataStoreService.PutExecutionType:output_type -> ml_metadata.PutExecutionTypeResponse - 19, // 144: ml_metadata.MetadataStoreService.PutContextType:output_type -> ml_metadata.PutContextTypeResponse - 17, // 145: ml_metadata.MetadataStoreService.PutTypes:output_type -> ml_metadata.PutTypesResponse - 5, // 146: ml_metadata.MetadataStoreService.PutArtifacts:output_type -> ml_metadata.PutArtifactsResponse - 9, // 147: ml_metadata.MetadataStoreService.PutExecutions:output_type -> ml_metadata.PutExecutionsResponse - 13, // 148: ml_metadata.MetadataStoreService.PutEvents:output_type -> ml_metadata.PutEventsResponse - 15, // 149: ml_metadata.MetadataStoreService.PutExecution:output_type -> ml_metadata.PutExecutionResponse - 21, // 150: ml_metadata.MetadataStoreService.PutContexts:output_type -> ml_metadata.PutContextsResponse - 23, // 151: ml_metadata.MetadataStoreService.PutAttributionsAndAssociations:output_type -> ml_metadata.PutAttributionsAndAssociationsResponse - 25, // 152: ml_metadata.MetadataStoreService.PutParentContexts:output_type -> ml_metadata.PutParentContextsResponse - 39, // 153: ml_metadata.MetadataStoreService.GetArtifactType:output_type -> ml_metadata.GetArtifactTypeResponse - 59, // 154: ml_metadata.MetadataStoreService.GetArtifactTypesByID:output_type -> ml_metadata.GetArtifactTypesByIDResponse - 41, // 155: ml_metadata.MetadataStoreService.GetArtifactTypes:output_type -> ml_metadata.GetArtifactTypesResponse - 53, // 156: ml_metadata.MetadataStoreService.GetExecutionType:output_type -> ml_metadata.GetExecutionTypeResponse - 61, // 157: ml_metadata.MetadataStoreService.GetExecutionTypesByID:output_type -> ml_metadata.GetExecutionTypesByIDResponse - 43, // 158: ml_metadata.MetadataStoreService.GetExecutionTypes:output_type -> ml_metadata.GetExecutionTypesResponse - 63, // 159: ml_metadata.MetadataStoreService.GetContextType:output_type -> ml_metadata.GetContextTypeResponse - 65, // 160: ml_metadata.MetadataStoreService.GetContextTypesByID:output_type -> ml_metadata.GetContextTypesByIDResponse - 45, // 161: ml_metadata.MetadataStoreService.GetContextTypes:output_type -> ml_metadata.GetContextTypesResponse - 33, // 162: ml_metadata.MetadataStoreService.GetArtifacts:output_type -> ml_metadata.GetArtifactsResponse - 37, // 163: ml_metadata.MetadataStoreService.GetExecutions:output_type -> ml_metadata.GetExecutionsResponse - 67, // 164: ml_metadata.MetadataStoreService.GetContexts:output_type -> ml_metadata.GetContextsResponse - 31, // 165: ml_metadata.MetadataStoreService.GetArtifactsByID:output_type -> ml_metadata.GetArtifactsByIDResponse - 51, // 166: ml_metadata.MetadataStoreService.GetExecutionsByID:output_type -> ml_metadata.GetExecutionsByIDResponse - 73, // 167: ml_metadata.MetadataStoreService.GetContextsByID:output_type -> ml_metadata.GetContextsByIDResponse - 27, // 168: ml_metadata.MetadataStoreService.GetArtifactsByType:output_type -> ml_metadata.GetArtifactsByTypeResponse - 47, // 169: ml_metadata.MetadataStoreService.GetExecutionsByType:output_type -> ml_metadata.GetExecutionsByTypeResponse - 69, // 170: ml_metadata.MetadataStoreService.GetContextsByType:output_type -> ml_metadata.GetContextsByTypeResponse - 29, // 171: ml_metadata.MetadataStoreService.GetArtifactByTypeAndName:output_type -> ml_metadata.GetArtifactByTypeAndNameResponse - 49, // 172: ml_metadata.MetadataStoreService.GetExecutionByTypeAndName:output_type -> ml_metadata.GetExecutionByTypeAndNameResponse - 71, // 173: ml_metadata.MetadataStoreService.GetContextByTypeAndName:output_type -> ml_metadata.GetContextByTypeAndNameResponse - 35, // 174: ml_metadata.MetadataStoreService.GetArtifactsByURI:output_type -> ml_metadata.GetArtifactsByURIResponse - 55, // 175: ml_metadata.MetadataStoreService.GetEventsByExecutionIDs:output_type -> ml_metadata.GetEventsByExecutionIDsResponse - 57, // 176: ml_metadata.MetadataStoreService.GetEventsByArtifactIDs:output_type -> ml_metadata.GetEventsByArtifactIDsResponse - 75, // 177: ml_metadata.MetadataStoreService.GetContextsByArtifact:output_type -> ml_metadata.GetContextsByArtifactResponse - 77, // 178: ml_metadata.MetadataStoreService.GetContextsByExecution:output_type -> ml_metadata.GetContextsByExecutionResponse - 79, // 179: ml_metadata.MetadataStoreService.GetParentContextsByContext:output_type -> ml_metadata.GetParentContextsByContextResponse - 81, // 180: ml_metadata.MetadataStoreService.GetChildrenContextsByContext:output_type -> ml_metadata.GetChildrenContextsByContextResponse - 83, // 181: ml_metadata.MetadataStoreService.GetArtifactsByContext:output_type -> ml_metadata.GetArtifactsByContextResponse - 85, // 182: ml_metadata.MetadataStoreService.GetExecutionsByContext:output_type -> ml_metadata.GetExecutionsByContextResponse - 87, // 183: ml_metadata.MetadataStoreService.GetLineageGraph:output_type -> ml_metadata.GetLineageGraphResponse - 142, // [142:184] is the sub-list for method output_type - 100, // [100:142] is the sub-list for method input_type - 100, // [100:100] is the sub-list for extension type_name - 100, // [100:100] is the sub-list for extension extendee - 0, // [0:100] is the sub-list for field type_name + 118, // 7: ml_metadata.PutArtifactsRequest.artifacts:type_name -> ml_metadata.Artifact + 109, // 8: ml_metadata.PutArtifactsRequest.options:type_name -> ml_metadata.PutArtifactsRequest.Options + 120, // 9: ml_metadata.PutArtifactsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 121, // 10: ml_metadata.PutArtifactsRequest.update_mask:type_name -> google.protobuf.FieldMask + 119, // 11: ml_metadata.PutArtifactTypeRequest.artifact_type:type_name -> ml_metadata.ArtifactType + 120, // 12: ml_metadata.PutArtifactTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 13: ml_metadata.PutExecutionsRequest.executions:type_name -> ml_metadata.Execution + 120, // 14: ml_metadata.PutExecutionsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 121, // 15: ml_metadata.PutExecutionsRequest.update_mask:type_name -> google.protobuf.FieldMask + 123, // 16: ml_metadata.PutExecutionTypeRequest.execution_type:type_name -> ml_metadata.ExecutionType + 120, // 17: ml_metadata.PutExecutionTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 124, // 18: ml_metadata.PutEventsRequest.events:type_name -> ml_metadata.Event + 120, // 19: ml_metadata.PutEventsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 20: ml_metadata.PutExecutionRequest.execution:type_name -> ml_metadata.Execution + 110, // 21: ml_metadata.PutExecutionRequest.artifact_event_pairs:type_name -> ml_metadata.PutExecutionRequest.ArtifactAndEvent + 125, // 22: ml_metadata.PutExecutionRequest.contexts:type_name -> ml_metadata.Context + 111, // 23: ml_metadata.PutExecutionRequest.options:type_name -> ml_metadata.PutExecutionRequest.Options + 120, // 24: ml_metadata.PutExecutionRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 25: ml_metadata.PutLineageSubgraphRequest.executions:type_name -> ml_metadata.Execution + 118, // 26: ml_metadata.PutLineageSubgraphRequest.artifacts:type_name -> ml_metadata.Artifact + 125, // 27: ml_metadata.PutLineageSubgraphRequest.contexts:type_name -> ml_metadata.Context + 112, // 28: ml_metadata.PutLineageSubgraphRequest.event_edges:type_name -> ml_metadata.PutLineageSubgraphRequest.EventEdge + 113, // 29: ml_metadata.PutLineageSubgraphRequest.options:type_name -> ml_metadata.PutLineageSubgraphRequest.Options + 120, // 30: ml_metadata.PutLineageSubgraphRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 119, // 31: ml_metadata.PutTypesRequest.artifact_types:type_name -> ml_metadata.ArtifactType + 123, // 32: ml_metadata.PutTypesRequest.execution_types:type_name -> ml_metadata.ExecutionType + 126, // 33: ml_metadata.PutTypesRequest.context_types:type_name -> ml_metadata.ContextType + 120, // 34: ml_metadata.PutTypesRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 126, // 35: ml_metadata.PutContextTypeRequest.context_type:type_name -> ml_metadata.ContextType + 120, // 36: ml_metadata.PutContextTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 37: ml_metadata.PutContextsRequest.contexts:type_name -> ml_metadata.Context + 120, // 38: ml_metadata.PutContextsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 121, // 39: ml_metadata.PutContextsRequest.update_mask:type_name -> google.protobuf.FieldMask + 127, // 40: ml_metadata.PutAttributionsAndAssociationsRequest.attributions:type_name -> ml_metadata.Attribution + 128, // 41: ml_metadata.PutAttributionsAndAssociationsRequest.associations:type_name -> ml_metadata.Association + 120, // 42: ml_metadata.PutAttributionsAndAssociationsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 129, // 43: ml_metadata.PutParentContextsRequest.parent_contexts:type_name -> ml_metadata.ParentContext + 120, // 44: ml_metadata.PutParentContextsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 130, // 45: ml_metadata.GetArtifactsByTypeRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 46: ml_metadata.GetArtifactsByTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 118, // 47: ml_metadata.GetArtifactsByTypeResponse.artifacts:type_name -> ml_metadata.Artifact + 120, // 48: ml_metadata.GetArtifactByTypeAndNameRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 118, // 49: ml_metadata.GetArtifactByTypeAndNameResponse.artifact:type_name -> ml_metadata.Artifact + 120, // 50: ml_metadata.GetArtifactsByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 118, // 51: ml_metadata.GetArtifactsByIDResponse.artifacts:type_name -> ml_metadata.Artifact + 119, // 52: ml_metadata.GetArtifactsByIDResponse.artifact_types:type_name -> ml_metadata.ArtifactType + 130, // 53: ml_metadata.GetArtifactsRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 54: ml_metadata.GetArtifactsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 118, // 55: ml_metadata.GetArtifactsResponse.artifacts:type_name -> ml_metadata.Artifact + 120, // 56: ml_metadata.GetArtifactsByURIRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 118, // 57: ml_metadata.GetArtifactsByURIResponse.artifacts:type_name -> ml_metadata.Artifact + 130, // 58: ml_metadata.GetExecutionsRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 59: ml_metadata.GetExecutionsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 60: ml_metadata.GetExecutionsResponse.executions:type_name -> ml_metadata.Execution + 120, // 61: ml_metadata.GetArtifactTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 119, // 62: ml_metadata.GetArtifactTypeResponse.artifact_type:type_name -> ml_metadata.ArtifactType + 120, // 63: ml_metadata.GetArtifactTypesRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 119, // 64: ml_metadata.GetArtifactTypesResponse.artifact_types:type_name -> ml_metadata.ArtifactType + 120, // 65: ml_metadata.GetExecutionTypesRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 123, // 66: ml_metadata.GetExecutionTypesResponse.execution_types:type_name -> ml_metadata.ExecutionType + 120, // 67: ml_metadata.GetContextTypesRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 126, // 68: ml_metadata.GetContextTypesResponse.context_types:type_name -> ml_metadata.ContextType + 120, // 69: ml_metadata.GetArtifactsByExternalIdsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 118, // 70: ml_metadata.GetArtifactsByExternalIdsResponse.artifacts:type_name -> ml_metadata.Artifact + 120, // 71: ml_metadata.GetExecutionsByExternalIdsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 72: ml_metadata.GetExecutionsByExternalIdsResponse.executions:type_name -> ml_metadata.Execution + 120, // 73: ml_metadata.GetContextsByExternalIdsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 74: ml_metadata.GetContextsByExternalIdsResponse.contexts:type_name -> ml_metadata.Context + 120, // 75: ml_metadata.GetArtifactTypesByExternalIdsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 119, // 76: ml_metadata.GetArtifactTypesByExternalIdsResponse.artifact_types:type_name -> ml_metadata.ArtifactType + 120, // 77: ml_metadata.GetExecutionTypesByExternalIdsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 123, // 78: ml_metadata.GetExecutionTypesByExternalIdsResponse.execution_types:type_name -> ml_metadata.ExecutionType + 120, // 79: ml_metadata.GetContextTypesByExternalIdsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 126, // 80: ml_metadata.GetContextTypesByExternalIdsResponse.context_types:type_name -> ml_metadata.ContextType + 130, // 81: ml_metadata.GetExecutionsByTypeRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 82: ml_metadata.GetExecutionsByTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 83: ml_metadata.GetExecutionsByTypeResponse.executions:type_name -> ml_metadata.Execution + 120, // 84: ml_metadata.GetExecutionByTypeAndNameRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 85: ml_metadata.GetExecutionByTypeAndNameResponse.execution:type_name -> ml_metadata.Execution + 120, // 86: ml_metadata.GetExecutionsByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 87: ml_metadata.GetExecutionsByIDResponse.executions:type_name -> ml_metadata.Execution + 120, // 88: ml_metadata.GetExecutionTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 123, // 89: ml_metadata.GetExecutionTypeResponse.execution_type:type_name -> ml_metadata.ExecutionType + 120, // 90: ml_metadata.GetEventsByExecutionIDsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 124, // 91: ml_metadata.GetEventsByExecutionIDsResponse.events:type_name -> ml_metadata.Event + 120, // 92: ml_metadata.GetEventsByArtifactIDsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 124, // 93: ml_metadata.GetEventsByArtifactIDsResponse.events:type_name -> ml_metadata.Event + 120, // 94: ml_metadata.GetArtifactTypesByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 119, // 95: ml_metadata.GetArtifactTypesByIDResponse.artifact_types:type_name -> ml_metadata.ArtifactType + 120, // 96: ml_metadata.GetExecutionTypesByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 123, // 97: ml_metadata.GetExecutionTypesByIDResponse.execution_types:type_name -> ml_metadata.ExecutionType + 120, // 98: ml_metadata.GetContextTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 126, // 99: ml_metadata.GetContextTypeResponse.context_type:type_name -> ml_metadata.ContextType + 120, // 100: ml_metadata.GetContextTypesByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 126, // 101: ml_metadata.GetContextTypesByIDResponse.context_types:type_name -> ml_metadata.ContextType + 130, // 102: ml_metadata.GetContextsRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 103: ml_metadata.GetContextsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 104: ml_metadata.GetContextsResponse.contexts:type_name -> ml_metadata.Context + 130, // 105: ml_metadata.GetContextsByTypeRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 106: ml_metadata.GetContextsByTypeRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 107: ml_metadata.GetContextsByTypeResponse.contexts:type_name -> ml_metadata.Context + 120, // 108: ml_metadata.GetContextByTypeAndNameRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 109: ml_metadata.GetContextByTypeAndNameResponse.context:type_name -> ml_metadata.Context + 120, // 110: ml_metadata.GetContextsByIDRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 111: ml_metadata.GetContextsByIDResponse.contexts:type_name -> ml_metadata.Context + 120, // 112: ml_metadata.GetContextsByArtifactRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 113: ml_metadata.GetContextsByArtifactResponse.contexts:type_name -> ml_metadata.Context + 120, // 114: ml_metadata.GetContextsByExecutionRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 115: ml_metadata.GetContextsByExecutionResponse.contexts:type_name -> ml_metadata.Context + 120, // 116: ml_metadata.GetParentContextsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 117: ml_metadata.GetParentContextsByContextResponse.contexts:type_name -> ml_metadata.Context + 120, // 118: ml_metadata.GetChildrenContextsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 125, // 119: ml_metadata.GetChildrenContextsByContextResponse.contexts:type_name -> ml_metadata.Context + 120, // 120: ml_metadata.GetParentContextsByContextsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 115, // 121: ml_metadata.GetParentContextsByContextsResponse.contexts:type_name -> ml_metadata.GetParentContextsByContextsResponse.ContextsEntry + 120, // 122: ml_metadata.GetChildrenContextsByContextsRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 117, // 123: ml_metadata.GetChildrenContextsByContextsResponse.contexts:type_name -> ml_metadata.GetChildrenContextsByContextsResponse.ContextsEntry + 130, // 124: ml_metadata.GetArtifactsByContextRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 125: ml_metadata.GetArtifactsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 118, // 126: ml_metadata.GetArtifactsByContextResponse.artifacts:type_name -> ml_metadata.Artifact + 130, // 127: ml_metadata.GetExecutionsByContextRequest.options:type_name -> ml_metadata.ListOperationOptions + 120, // 128: ml_metadata.GetExecutionsByContextRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 122, // 129: ml_metadata.GetExecutionsByContextResponse.executions:type_name -> ml_metadata.Execution + 120, // 130: ml_metadata.GetExecutionsByContextResponse.transaction_options:type_name -> ml_metadata.TransactionOptions + 131, // 131: ml_metadata.GetLineageGraphRequest.options:type_name -> ml_metadata.LineageGraphQueryOptions + 120, // 132: ml_metadata.GetLineageGraphRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 132, // 133: ml_metadata.GetLineageGraphResponse.subgraph:type_name -> ml_metadata.LineageGraph + 133, // 134: ml_metadata.GetLineageSubgraphRequest.lineage_subgraph_query_options:type_name -> ml_metadata.LineageSubgraphQueryOptions + 121, // 135: ml_metadata.GetLineageSubgraphRequest.read_mask:type_name -> google.protobuf.FieldMask + 120, // 136: ml_metadata.GetLineageSubgraphRequest.transaction_options:type_name -> ml_metadata.TransactionOptions + 132, // 137: ml_metadata.GetLineageSubgraphResponse.lineage_subgraph:type_name -> ml_metadata.LineageGraph + 3, // 138: ml_metadata.ArtifactStructMap.PropertiesEntry.value:type_name -> ml_metadata.ArtifactStruct + 118, // 139: ml_metadata.PutExecutionRequest.ArtifactAndEvent.artifact:type_name -> ml_metadata.Artifact + 124, // 140: ml_metadata.PutExecutionRequest.ArtifactAndEvent.event:type_name -> ml_metadata.Event + 124, // 141: ml_metadata.PutLineageSubgraphRequest.EventEdge.event:type_name -> ml_metadata.Event + 125, // 142: ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild.parent_contexts:type_name -> ml_metadata.Context + 114, // 143: ml_metadata.GetParentContextsByContextsResponse.ContextsEntry.value:type_name -> ml_metadata.GetParentContextsByContextsResponse.ParentContextsPerChild + 125, // 144: ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent.children_contexts:type_name -> ml_metadata.Context + 116, // 145: ml_metadata.GetChildrenContextsByContextsResponse.ContextsEntry.value:type_name -> ml_metadata.GetChildrenContextsByContextsResponse.ChildrenContextsPerParent + 6, // 146: ml_metadata.MetadataStoreService.PutArtifactType:input_type -> ml_metadata.PutArtifactTypeRequest + 10, // 147: ml_metadata.MetadataStoreService.PutExecutionType:input_type -> ml_metadata.PutExecutionTypeRequest + 20, // 148: ml_metadata.MetadataStoreService.PutContextType:input_type -> ml_metadata.PutContextTypeRequest + 18, // 149: ml_metadata.MetadataStoreService.PutTypes:input_type -> ml_metadata.PutTypesRequest + 4, // 150: ml_metadata.MetadataStoreService.PutArtifacts:input_type -> ml_metadata.PutArtifactsRequest + 8, // 151: ml_metadata.MetadataStoreService.PutExecutions:input_type -> ml_metadata.PutExecutionsRequest + 12, // 152: ml_metadata.MetadataStoreService.PutEvents:input_type -> ml_metadata.PutEventsRequest + 14, // 153: ml_metadata.MetadataStoreService.PutExecution:input_type -> ml_metadata.PutExecutionRequest + 16, // 154: ml_metadata.MetadataStoreService.PutLineageSubgraph:input_type -> ml_metadata.PutLineageSubgraphRequest + 22, // 155: ml_metadata.MetadataStoreService.PutContexts:input_type -> ml_metadata.PutContextsRequest + 24, // 156: ml_metadata.MetadataStoreService.PutAttributionsAndAssociations:input_type -> ml_metadata.PutAttributionsAndAssociationsRequest + 26, // 157: ml_metadata.MetadataStoreService.PutParentContexts:input_type -> ml_metadata.PutParentContextsRequest + 40, // 158: ml_metadata.MetadataStoreService.GetArtifactType:input_type -> ml_metadata.GetArtifactTypeRequest + 72, // 159: ml_metadata.MetadataStoreService.GetArtifactTypesByID:input_type -> ml_metadata.GetArtifactTypesByIDRequest + 42, // 160: ml_metadata.MetadataStoreService.GetArtifactTypes:input_type -> ml_metadata.GetArtifactTypesRequest + 66, // 161: ml_metadata.MetadataStoreService.GetExecutionType:input_type -> ml_metadata.GetExecutionTypeRequest + 74, // 162: ml_metadata.MetadataStoreService.GetExecutionTypesByID:input_type -> ml_metadata.GetExecutionTypesByIDRequest + 44, // 163: ml_metadata.MetadataStoreService.GetExecutionTypes:input_type -> ml_metadata.GetExecutionTypesRequest + 76, // 164: ml_metadata.MetadataStoreService.GetContextType:input_type -> ml_metadata.GetContextTypeRequest + 78, // 165: ml_metadata.MetadataStoreService.GetContextTypesByID:input_type -> ml_metadata.GetContextTypesByIDRequest + 46, // 166: ml_metadata.MetadataStoreService.GetContextTypes:input_type -> ml_metadata.GetContextTypesRequest + 34, // 167: ml_metadata.MetadataStoreService.GetArtifacts:input_type -> ml_metadata.GetArtifactsRequest + 38, // 168: ml_metadata.MetadataStoreService.GetExecutions:input_type -> ml_metadata.GetExecutionsRequest + 80, // 169: ml_metadata.MetadataStoreService.GetContexts:input_type -> ml_metadata.GetContextsRequest + 32, // 170: ml_metadata.MetadataStoreService.GetArtifactsByID:input_type -> ml_metadata.GetArtifactsByIDRequest + 64, // 171: ml_metadata.MetadataStoreService.GetExecutionsByID:input_type -> ml_metadata.GetExecutionsByIDRequest + 86, // 172: ml_metadata.MetadataStoreService.GetContextsByID:input_type -> ml_metadata.GetContextsByIDRequest + 28, // 173: ml_metadata.MetadataStoreService.GetArtifactsByType:input_type -> ml_metadata.GetArtifactsByTypeRequest + 60, // 174: ml_metadata.MetadataStoreService.GetExecutionsByType:input_type -> ml_metadata.GetExecutionsByTypeRequest + 82, // 175: ml_metadata.MetadataStoreService.GetContextsByType:input_type -> ml_metadata.GetContextsByTypeRequest + 30, // 176: ml_metadata.MetadataStoreService.GetArtifactByTypeAndName:input_type -> ml_metadata.GetArtifactByTypeAndNameRequest + 62, // 177: ml_metadata.MetadataStoreService.GetExecutionByTypeAndName:input_type -> ml_metadata.GetExecutionByTypeAndNameRequest + 84, // 178: ml_metadata.MetadataStoreService.GetContextByTypeAndName:input_type -> ml_metadata.GetContextByTypeAndNameRequest + 36, // 179: ml_metadata.MetadataStoreService.GetArtifactsByURI:input_type -> ml_metadata.GetArtifactsByURIRequest + 68, // 180: ml_metadata.MetadataStoreService.GetEventsByExecutionIDs:input_type -> ml_metadata.GetEventsByExecutionIDsRequest + 70, // 181: ml_metadata.MetadataStoreService.GetEventsByArtifactIDs:input_type -> ml_metadata.GetEventsByArtifactIDsRequest + 48, // 182: ml_metadata.MetadataStoreService.GetArtifactsByExternalIds:input_type -> ml_metadata.GetArtifactsByExternalIdsRequest + 50, // 183: ml_metadata.MetadataStoreService.GetExecutionsByExternalIds:input_type -> ml_metadata.GetExecutionsByExternalIdsRequest + 52, // 184: ml_metadata.MetadataStoreService.GetContextsByExternalIds:input_type -> ml_metadata.GetContextsByExternalIdsRequest + 54, // 185: ml_metadata.MetadataStoreService.GetArtifactTypesByExternalIds:input_type -> ml_metadata.GetArtifactTypesByExternalIdsRequest + 56, // 186: ml_metadata.MetadataStoreService.GetExecutionTypesByExternalIds:input_type -> ml_metadata.GetExecutionTypesByExternalIdsRequest + 58, // 187: ml_metadata.MetadataStoreService.GetContextTypesByExternalIds:input_type -> ml_metadata.GetContextTypesByExternalIdsRequest + 88, // 188: ml_metadata.MetadataStoreService.GetContextsByArtifact:input_type -> ml_metadata.GetContextsByArtifactRequest + 90, // 189: ml_metadata.MetadataStoreService.GetContextsByExecution:input_type -> ml_metadata.GetContextsByExecutionRequest + 92, // 190: ml_metadata.MetadataStoreService.GetParentContextsByContext:input_type -> ml_metadata.GetParentContextsByContextRequest + 94, // 191: ml_metadata.MetadataStoreService.GetChildrenContextsByContext:input_type -> ml_metadata.GetChildrenContextsByContextRequest + 96, // 192: ml_metadata.MetadataStoreService.GetParentContextsByContexts:input_type -> ml_metadata.GetParentContextsByContextsRequest + 98, // 193: ml_metadata.MetadataStoreService.GetChildrenContextsByContexts:input_type -> ml_metadata.GetChildrenContextsByContextsRequest + 100, // 194: ml_metadata.MetadataStoreService.GetArtifactsByContext:input_type -> ml_metadata.GetArtifactsByContextRequest + 102, // 195: ml_metadata.MetadataStoreService.GetExecutionsByContext:input_type -> ml_metadata.GetExecutionsByContextRequest + 104, // 196: ml_metadata.MetadataStoreService.GetLineageGraph:input_type -> ml_metadata.GetLineageGraphRequest + 106, // 197: ml_metadata.MetadataStoreService.GetLineageSubgraph:input_type -> ml_metadata.GetLineageSubgraphRequest + 7, // 198: ml_metadata.MetadataStoreService.PutArtifactType:output_type -> ml_metadata.PutArtifactTypeResponse + 11, // 199: ml_metadata.MetadataStoreService.PutExecutionType:output_type -> ml_metadata.PutExecutionTypeResponse + 21, // 200: ml_metadata.MetadataStoreService.PutContextType:output_type -> ml_metadata.PutContextTypeResponse + 19, // 201: ml_metadata.MetadataStoreService.PutTypes:output_type -> ml_metadata.PutTypesResponse + 5, // 202: ml_metadata.MetadataStoreService.PutArtifacts:output_type -> ml_metadata.PutArtifactsResponse + 9, // 203: ml_metadata.MetadataStoreService.PutExecutions:output_type -> ml_metadata.PutExecutionsResponse + 13, // 204: ml_metadata.MetadataStoreService.PutEvents:output_type -> ml_metadata.PutEventsResponse + 15, // 205: ml_metadata.MetadataStoreService.PutExecution:output_type -> ml_metadata.PutExecutionResponse + 17, // 206: ml_metadata.MetadataStoreService.PutLineageSubgraph:output_type -> ml_metadata.PutLineageSubgraphResponse + 23, // 207: ml_metadata.MetadataStoreService.PutContexts:output_type -> ml_metadata.PutContextsResponse + 25, // 208: ml_metadata.MetadataStoreService.PutAttributionsAndAssociations:output_type -> ml_metadata.PutAttributionsAndAssociationsResponse + 27, // 209: ml_metadata.MetadataStoreService.PutParentContexts:output_type -> ml_metadata.PutParentContextsResponse + 41, // 210: ml_metadata.MetadataStoreService.GetArtifactType:output_type -> ml_metadata.GetArtifactTypeResponse + 73, // 211: ml_metadata.MetadataStoreService.GetArtifactTypesByID:output_type -> ml_metadata.GetArtifactTypesByIDResponse + 43, // 212: ml_metadata.MetadataStoreService.GetArtifactTypes:output_type -> ml_metadata.GetArtifactTypesResponse + 67, // 213: ml_metadata.MetadataStoreService.GetExecutionType:output_type -> ml_metadata.GetExecutionTypeResponse + 75, // 214: ml_metadata.MetadataStoreService.GetExecutionTypesByID:output_type -> ml_metadata.GetExecutionTypesByIDResponse + 45, // 215: ml_metadata.MetadataStoreService.GetExecutionTypes:output_type -> ml_metadata.GetExecutionTypesResponse + 77, // 216: ml_metadata.MetadataStoreService.GetContextType:output_type -> ml_metadata.GetContextTypeResponse + 79, // 217: ml_metadata.MetadataStoreService.GetContextTypesByID:output_type -> ml_metadata.GetContextTypesByIDResponse + 47, // 218: ml_metadata.MetadataStoreService.GetContextTypes:output_type -> ml_metadata.GetContextTypesResponse + 35, // 219: ml_metadata.MetadataStoreService.GetArtifacts:output_type -> ml_metadata.GetArtifactsResponse + 39, // 220: ml_metadata.MetadataStoreService.GetExecutions:output_type -> ml_metadata.GetExecutionsResponse + 81, // 221: ml_metadata.MetadataStoreService.GetContexts:output_type -> ml_metadata.GetContextsResponse + 33, // 222: ml_metadata.MetadataStoreService.GetArtifactsByID:output_type -> ml_metadata.GetArtifactsByIDResponse + 65, // 223: ml_metadata.MetadataStoreService.GetExecutionsByID:output_type -> ml_metadata.GetExecutionsByIDResponse + 87, // 224: ml_metadata.MetadataStoreService.GetContextsByID:output_type -> ml_metadata.GetContextsByIDResponse + 29, // 225: ml_metadata.MetadataStoreService.GetArtifactsByType:output_type -> ml_metadata.GetArtifactsByTypeResponse + 61, // 226: ml_metadata.MetadataStoreService.GetExecutionsByType:output_type -> ml_metadata.GetExecutionsByTypeResponse + 83, // 227: ml_metadata.MetadataStoreService.GetContextsByType:output_type -> ml_metadata.GetContextsByTypeResponse + 31, // 228: ml_metadata.MetadataStoreService.GetArtifactByTypeAndName:output_type -> ml_metadata.GetArtifactByTypeAndNameResponse + 63, // 229: ml_metadata.MetadataStoreService.GetExecutionByTypeAndName:output_type -> ml_metadata.GetExecutionByTypeAndNameResponse + 85, // 230: ml_metadata.MetadataStoreService.GetContextByTypeAndName:output_type -> ml_metadata.GetContextByTypeAndNameResponse + 37, // 231: ml_metadata.MetadataStoreService.GetArtifactsByURI:output_type -> ml_metadata.GetArtifactsByURIResponse + 69, // 232: ml_metadata.MetadataStoreService.GetEventsByExecutionIDs:output_type -> ml_metadata.GetEventsByExecutionIDsResponse + 71, // 233: ml_metadata.MetadataStoreService.GetEventsByArtifactIDs:output_type -> ml_metadata.GetEventsByArtifactIDsResponse + 49, // 234: ml_metadata.MetadataStoreService.GetArtifactsByExternalIds:output_type -> ml_metadata.GetArtifactsByExternalIdsResponse + 51, // 235: ml_metadata.MetadataStoreService.GetExecutionsByExternalIds:output_type -> ml_metadata.GetExecutionsByExternalIdsResponse + 53, // 236: ml_metadata.MetadataStoreService.GetContextsByExternalIds:output_type -> ml_metadata.GetContextsByExternalIdsResponse + 55, // 237: ml_metadata.MetadataStoreService.GetArtifactTypesByExternalIds:output_type -> ml_metadata.GetArtifactTypesByExternalIdsResponse + 57, // 238: ml_metadata.MetadataStoreService.GetExecutionTypesByExternalIds:output_type -> ml_metadata.GetExecutionTypesByExternalIdsResponse + 59, // 239: ml_metadata.MetadataStoreService.GetContextTypesByExternalIds:output_type -> ml_metadata.GetContextTypesByExternalIdsResponse + 89, // 240: ml_metadata.MetadataStoreService.GetContextsByArtifact:output_type -> ml_metadata.GetContextsByArtifactResponse + 91, // 241: ml_metadata.MetadataStoreService.GetContextsByExecution:output_type -> ml_metadata.GetContextsByExecutionResponse + 93, // 242: ml_metadata.MetadataStoreService.GetParentContextsByContext:output_type -> ml_metadata.GetParentContextsByContextResponse + 95, // 243: ml_metadata.MetadataStoreService.GetChildrenContextsByContext:output_type -> ml_metadata.GetChildrenContextsByContextResponse + 97, // 244: ml_metadata.MetadataStoreService.GetParentContextsByContexts:output_type -> ml_metadata.GetParentContextsByContextsResponse + 99, // 245: ml_metadata.MetadataStoreService.GetChildrenContextsByContexts:output_type -> ml_metadata.GetChildrenContextsByContextsResponse + 101, // 246: ml_metadata.MetadataStoreService.GetArtifactsByContext:output_type -> ml_metadata.GetArtifactsByContextResponse + 103, // 247: ml_metadata.MetadataStoreService.GetExecutionsByContext:output_type -> ml_metadata.GetExecutionsByContextResponse + 105, // 248: ml_metadata.MetadataStoreService.GetLineageGraph:output_type -> ml_metadata.GetLineageGraphResponse + 107, // 249: ml_metadata.MetadataStoreService.GetLineageSubgraph:output_type -> ml_metadata.GetLineageSubgraphResponse + 198, // [198:250] is the sub-list for method output_type + 146, // [146:198] is the sub-list for method input_type + 146, // [146:146] is the sub-list for extension type_name + 146, // [146:146] is the sub-list for extension extendee + 0, // [0:146] is the sub-list for field type_name } func init() { file_ml_metadata_proto_metadata_store_service_proto_init() } @@ -6630,8 +8812,104 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactStructMap); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ArtifactStructMap); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ArtifactStructList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ArtifactStruct); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutArtifactsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutArtifactsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutArtifactTypeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutArtifactTypeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionsResponse); i { case 0: return &v.state case 1: @@ -6642,8 +8920,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactStructList); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionTypeRequest); i { case 0: return &v.state case 1: @@ -6654,8 +8932,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArtifactStruct); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionTypeResponse); i { case 0: return &v.state case 1: @@ -6666,8 +8944,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutArtifactsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutEventsRequest); i { case 0: return &v.state case 1: @@ -6678,8 +8956,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutArtifactsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutEventsResponse); i { case 0: return &v.state case 1: @@ -6690,8 +8968,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutArtifactTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionRequest); i { case 0: return &v.state case 1: @@ -6702,8 +8980,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutArtifactTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionResponse); i { case 0: return &v.state case 1: @@ -6714,8 +8992,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutLineageSubgraphRequest); i { case 0: return &v.state case 1: @@ -6726,8 +9004,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutLineageSubgraphResponse); i { case 0: return &v.state case 1: @@ -6738,8 +9016,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutTypesRequest); i { case 0: return &v.state case 1: @@ -6750,8 +9028,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutTypesResponse); i { case 0: return &v.state case 1: @@ -6762,8 +9040,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutEventsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutContextTypeRequest); i { case 0: return &v.state case 1: @@ -6774,8 +9052,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutEventsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutContextTypeResponse); i { case 0: return &v.state case 1: @@ -6786,8 +9064,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutContextsRequest); i { case 0: return &v.state case 1: @@ -6798,8 +9076,200 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutContextsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutAttributionsAndAssociationsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutAttributionsAndAssociationsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutParentContextsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutParentContextsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByTypeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByTypeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactByTypeAndNameRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactByTypeAndNameResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByIDRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByIDResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByURIRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByURIResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsResponse); i { case 0: return &v.state case 1: @@ -6810,8 +9280,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutTypesRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypeRequest); i { case 0: return &v.state case 1: @@ -6822,8 +9292,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutTypesResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypeResponse); i { case 0: return &v.state case 1: @@ -6834,8 +9304,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutContextTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypesRequest); i { case 0: return &v.state case 1: @@ -6846,8 +9316,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutContextTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypesResponse); i { case 0: return &v.state case 1: @@ -6858,8 +9328,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutContextsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypesRequest); i { case 0: return &v.state case 1: @@ -6870,8 +9340,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutContextsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypesResponse); i { case 0: return &v.state case 1: @@ -6882,8 +9352,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutAttributionsAndAssociationsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypesRequest); i { case 0: return &v.state case 1: @@ -6894,8 +9364,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutAttributionsAndAssociationsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypesResponse); i { case 0: return &v.state case 1: @@ -6906,8 +9376,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutParentContextsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByExternalIdsRequest); i { case 0: return &v.state case 1: @@ -6918,8 +9388,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutParentContextsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByExternalIdsResponse); i { case 0: return &v.state case 1: @@ -6930,8 +9400,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByExternalIdsRequest); i { case 0: return &v.state case 1: @@ -6942,8 +9412,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByExternalIdsResponse); i { case 0: return &v.state case 1: @@ -6954,8 +9424,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactByTypeAndNameRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByExternalIdsRequest); i { case 0: return &v.state case 1: @@ -6966,8 +9436,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactByTypeAndNameResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByExternalIdsResponse); i { case 0: return &v.state case 1: @@ -6978,8 +9448,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByIDRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypesByExternalIdsRequest); i { case 0: return &v.state case 1: @@ -6990,8 +9460,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByIDResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypesByExternalIdsResponse); i { case 0: return &v.state case 1: @@ -7002,8 +9472,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypesByExternalIdsRequest); i { case 0: return &v.state case 1: @@ -7014,8 +9484,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypesByExternalIdsResponse); i { case 0: return &v.state case 1: @@ -7026,8 +9496,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByURIRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypesByExternalIdsRequest); i { case 0: return &v.state case 1: @@ -7038,8 +9508,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByURIResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypesByExternalIdsResponse); i { case 0: return &v.state case 1: @@ -7050,8 +9520,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByTypeRequest); i { case 0: return &v.state case 1: @@ -7062,8 +9532,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByTypeResponse); i { case 0: return &v.state case 1: @@ -7074,8 +9544,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionByTypeAndNameRequest); i { case 0: return &v.state case 1: @@ -7086,8 +9556,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionByTypeAndNameResponse); i { case 0: return &v.state case 1: @@ -7098,8 +9568,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactTypesRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByIDRequest); i { case 0: return &v.state case 1: @@ -7110,8 +9580,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactTypesResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByIDResponse); i { case 0: return &v.state case 1: @@ -7122,8 +9592,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionTypesRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypeRequest); i { case 0: return &v.state case 1: @@ -7134,8 +9604,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionTypesResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypeResponse); i { case 0: return &v.state case 1: @@ -7146,8 +9616,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextTypesRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetEventsByExecutionIDsRequest); i { case 0: return &v.state case 1: @@ -7158,8 +9628,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextTypesResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetEventsByExecutionIDsResponse); i { case 0: return &v.state case 1: @@ -7170,8 +9640,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsByTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetEventsByArtifactIDsRequest); i { case 0: return &v.state case 1: @@ -7182,8 +9652,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsByTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetEventsByArtifactIDsResponse); i { case 0: return &v.state case 1: @@ -7194,8 +9664,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionByTypeAndNameRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypesByIDRequest); i { case 0: return &v.state case 1: @@ -7206,8 +9676,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionByTypeAndNameResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactTypesByIDResponse); i { case 0: return &v.state case 1: @@ -7218,8 +9688,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsByIDRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypesByIDRequest); i { case 0: return &v.state case 1: @@ -7230,8 +9700,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsByIDResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionTypesByIDResponse); i { case 0: return &v.state case 1: @@ -7242,8 +9712,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypeRequest); i { case 0: return &v.state case 1: @@ -7254,8 +9724,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypeResponse); i { case 0: return &v.state case 1: @@ -7266,8 +9736,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetEventsByExecutionIDsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypesByIDRequest); i { case 0: return &v.state case 1: @@ -7278,8 +9748,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetEventsByExecutionIDsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextTypesByIDResponse); i { case 0: return &v.state case 1: @@ -7290,8 +9760,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetEventsByArtifactIDsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsRequest); i { case 0: return &v.state case 1: @@ -7302,8 +9772,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetEventsByArtifactIDsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsResponse); i { case 0: return &v.state case 1: @@ -7314,8 +9784,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactTypesByIDRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByTypeRequest); i { case 0: return &v.state case 1: @@ -7326,8 +9796,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactTypesByIDResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByTypeResponse); i { case 0: return &v.state case 1: @@ -7338,8 +9808,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionTypesByIDRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextByTypeAndNameRequest); i { case 0: return &v.state case 1: @@ -7350,8 +9820,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionTypesByIDResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextByTypeAndNameResponse); i { case 0: return &v.state case 1: @@ -7362,8 +9832,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByIDRequest); i { case 0: return &v.state case 1: @@ -7374,8 +9844,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[87].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByIDResponse); i { case 0: return &v.state case 1: @@ -7386,8 +9856,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextTypesByIDRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[88].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByArtifactRequest); i { case 0: return &v.state case 1: @@ -7398,8 +9868,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextTypesByIDResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[89].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByArtifactResponse); i { case 0: return &v.state case 1: @@ -7410,8 +9880,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[90].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByExecutionRequest); i { case 0: return &v.state case 1: @@ -7422,8 +9892,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[91].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetContextsByExecutionResponse); i { case 0: return &v.state case 1: @@ -7434,8 +9904,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByTypeRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[92].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetParentContextsByContextRequest); i { case 0: return &v.state case 1: @@ -7446,8 +9916,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByTypeResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[93].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetParentContextsByContextResponse); i { case 0: return &v.state case 1: @@ -7458,8 +9928,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextByTypeAndNameRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[94].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChildrenContextsByContextRequest); i { case 0: return &v.state case 1: @@ -7470,8 +9940,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextByTypeAndNameResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[95].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChildrenContextsByContextResponse); i { case 0: return &v.state case 1: @@ -7482,8 +9952,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByIDRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[96].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetParentContextsByContextsRequest); i { case 0: return &v.state case 1: @@ -7494,8 +9964,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByIDResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[97].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetParentContextsByContextsResponse); i { case 0: return &v.state case 1: @@ -7506,8 +9976,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByArtifactRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[98].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChildrenContextsByContextsRequest); i { case 0: return &v.state case 1: @@ -7518,8 +9988,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByArtifactResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[99].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChildrenContextsByContextsResponse); i { case 0: return &v.state case 1: @@ -7530,8 +10000,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByExecutionRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[100].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByContextRequest); i { case 0: return &v.state case 1: @@ -7542,8 +10012,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetContextsByExecutionResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[101].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetArtifactsByContextResponse); i { case 0: return &v.state case 1: @@ -7554,8 +10024,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetParentContextsByContextRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[102].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByContextRequest); i { case 0: return &v.state case 1: @@ -7566,8 +10036,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetParentContextsByContextResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[103].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetExecutionsByContextResponse); i { case 0: return &v.state case 1: @@ -7578,8 +10048,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChildrenContextsByContextRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[104].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetLineageGraphRequest); i { case 0: return &v.state case 1: @@ -7590,8 +10060,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetChildrenContextsByContextResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[105].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetLineageGraphResponse); i { case 0: return &v.state case 1: @@ -7602,8 +10072,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByContextRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[106].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetLineageSubgraphRequest); i { case 0: return &v.state case 1: @@ -7614,8 +10084,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetArtifactsByContextResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[107].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetLineageSubgraphResponse); i { case 0: return &v.state case 1: @@ -7626,8 +10096,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsByContextRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[109].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutArtifactsRequest_Options); i { case 0: return &v.state case 1: @@ -7638,8 +10108,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[85].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetExecutionsByContextResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[110].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionRequest_ArtifactAndEvent); i { case 0: return &v.state case 1: @@ -7650,8 +10120,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[86].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetLineageGraphRequest); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[111].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutExecutionRequest_Options); i { case 0: return &v.state case 1: @@ -7662,8 +10132,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[87].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetLineageGraphResponse); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[112].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutLineageSubgraphRequest_EventEdge); i { case 0: return &v.state case 1: @@ -7674,8 +10144,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[89].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutArtifactsRequest_Options); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[113].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PutLineageSubgraphRequest_Options); i { case 0: return &v.state case 1: @@ -7686,8 +10156,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[90].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionRequest_ArtifactAndEvent); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[114].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetParentContextsByContextsResponse_ParentContextsPerChild); i { case 0: return &v.state case 1: @@ -7698,8 +10168,8 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { return nil } } - file_ml_metadata_proto_metadata_store_service_proto_msgTypes[91].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PutExecutionRequest_Options); i { + file_ml_metadata_proto_metadata_store_service_proto_msgTypes[116].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChildrenContextsByContextsResponse_ChildrenContextsPerParent); i { case 0: return &v.state case 1: @@ -7722,7 +10192,7 @@ func file_ml_metadata_proto_metadata_store_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_ml_metadata_proto_metadata_store_service_proto_rawDesc, NumEnums: 0, - NumMessages: 92, + NumMessages: 118, NumExtensions: 0, NumServices: 1, }, diff --git a/third_party/ml-metadata/go/ml_metadata/metadata_store_service_grpc.pb.go b/third_party/ml-metadata/go/ml_metadata/metadata_store_service_grpc.pb.go index a9426e0102..9417ff46a2 100644 --- a/third_party/ml-metadata/go/ml_metadata/metadata_store_service_grpc.pb.go +++ b/third_party/ml-metadata/go/ml_metadata/metadata_store_service_grpc.pb.go @@ -83,6 +83,10 @@ type MetadataStoreServiceClient interface { // For new artifacts, type must be specified. // For old artifacts, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated artifacts will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // artifacts: A list of artifacts to insert or update. // @@ -96,6 +100,10 @@ type MetadataStoreServiceClient interface { // For new executions, type must be specified. // For old executions, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated executions will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // executions: A list of executions to insert or update. // @@ -107,6 +115,10 @@ type MetadataStoreServiceClient interface { // // The execution_id and artifact_id must already exist. // Once created, events cannot be modified. + // AlreadyExists error will be raised if duplicated events are found. + // + // It is not guaranteed that the created or updated events will share the + // same `milliseconds_since_epoch` timestamps. // // Args: // events: A list of events to insert or update. @@ -117,9 +129,16 @@ type MetadataStoreServiceClient interface { // input/output Event. The `contexts` describe the associations of the // execution and the attributions of the artifacts. // - // If an execution_id, artifact_id or context_id is specified, it is an - // update, otherwise it does an insertion. For insertion, type must be - // specified. + // If an execution_id is specified, it is an update on the corresponding + // execution, otherwise it does an insertion. + // For insertion, type must be specified. Same rule applies to artifacts + // and contexts in the request. Corresponding errors may raised. For example: + // AlreadyExists error will be raised if duplicated executions, artifacts + // or events are found. + // + // It is not guaranteed that the created or updated executions, artifacts, + // contexts and events will share the same `create_time_since_epoch`, + // `last_update_time_since_epoch`, or `milliseconds_since_epoch` timestamps. // // Args: // execution: An execution to insert or update. @@ -130,6 +149,36 @@ type MetadataStoreServiceClient interface { // An execution id and a list of artifacts and contexts ids index-aligned // with the input. PutExecution(ctx context.Context, in *PutExecutionRequest, opts ...grpc.CallOption) (*PutExecutionResponse, error) + // Inserts or updates a lineage subgraph (i.e. a collection of event edges + // and its executions, artifacts, and related contexts) atomically. The + // `event_edges` include an Event and the indices of the corresponding + // execution and artifact from the input list of executions and artifacts. The + // `contexts` describe the associations of the Execution and the attributions + // of the Artifact. + // + // If an execution_id is specified, it is an update on the corresponding + // Execution, otherwise it does an insertion. For insertion, type must be + // specified. These rules apply to Artifacts and Contexts as well. + // Corresponding errors may be raised. For example: AlreadyExists error will + // be raised if duplicated executions, artifacts, or events are found. + // + // It is not guaranteed that the created or updated executions, artifacts, + // contexts and events will share the same `create_time_since_epoch`, + // `last_update_time_since_epoch`, or `milliseconds_since_epoch` timestamps. + // + // Args: + // executions: A list of executions to insert or update. + // artifacts: A list of artifacts to insert or update. + // contexts: A list of contexts to insert and/or create associations and + // attributions with. + // event_edges: A list of events to insert with the indices of the + // corresponding execution and artifact from the input lists of + // executions and artifacts. + // + // Returns: + // Lists of execution, artifact, and context ids index-aligned with the + // inputs. + PutLineageSubgraph(ctx context.Context, in *PutLineageSubgraphRequest, opts ...grpc.CallOption) (*PutLineageSubgraphResponse, error) // Inserts or updates contexts in database and returns a list of context ids. // // If an context_id is specified for a context, it is an update. @@ -137,6 +186,10 @@ type MetadataStoreServiceClient interface { // For new contexts, type must be specified. // For old contexts, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated contexts will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // contexts: A list of contexts to insert or update. // @@ -229,6 +282,18 @@ type MetadataStoreServiceClient interface { GetEventsByExecutionIDs(ctx context.Context, in *GetEventsByExecutionIDsRequest, opts ...grpc.CallOption) (*GetEventsByExecutionIDsResponse, error) // Gets all events with matching artifact ids. GetEventsByArtifactIDs(ctx context.Context, in *GetEventsByArtifactIDsRequest, opts ...grpc.CallOption) (*GetEventsByArtifactIDsResponse, error) + // Gets all the artifacts with matching external ids. + GetArtifactsByExternalIds(ctx context.Context, in *GetArtifactsByExternalIdsRequest, opts ...grpc.CallOption) (*GetArtifactsByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetExecutionsByExternalIds(ctx context.Context, in *GetExecutionsByExternalIdsRequest, opts ...grpc.CallOption) (*GetExecutionsByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetContextsByExternalIds(ctx context.Context, in *GetContextsByExternalIdsRequest, opts ...grpc.CallOption) (*GetContextsByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetArtifactTypesByExternalIds(ctx context.Context, in *GetArtifactTypesByExternalIdsRequest, opts ...grpc.CallOption) (*GetArtifactTypesByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetExecutionTypesByExternalIds(ctx context.Context, in *GetExecutionTypesByExternalIdsRequest, opts ...grpc.CallOption) (*GetExecutionTypesByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetContextTypesByExternalIds(ctx context.Context, in *GetContextTypesByExternalIdsRequest, opts ...grpc.CallOption) (*GetContextTypesByExternalIdsResponse, error) // Gets all context that an artifact is attributed to. GetContextsByArtifact(ctx context.Context, in *GetContextsByArtifactRequest, opts ...grpc.CallOption) (*GetContextsByArtifactResponse, error) // Gets all context that an execution is associated with. @@ -237,14 +302,26 @@ type MetadataStoreServiceClient interface { GetParentContextsByContext(ctx context.Context, in *GetParentContextsByContextRequest, opts ...grpc.CallOption) (*GetParentContextsByContextResponse, error) // Gets all children contexts that a context is related. GetChildrenContextsByContext(ctx context.Context, in *GetChildrenContextsByContextRequest, opts ...grpc.CallOption) (*GetChildrenContextsByContextResponse, error) + // Batch getting all the parent contexts that a list of contexts are related. + GetParentContextsByContexts(ctx context.Context, in *GetParentContextsByContextsRequest, opts ...grpc.CallOption) (*GetParentContextsByContextsResponse, error) + // Batch getting all the children contexts that a list of contexts are + // related. + GetChildrenContextsByContexts(ctx context.Context, in *GetChildrenContextsByContextsRequest, opts ...grpc.CallOption) (*GetChildrenContextsByContextsResponse, error) // Gets all direct artifacts that a context attributes to. GetArtifactsByContext(ctx context.Context, in *GetArtifactsByContextRequest, opts ...grpc.CallOption) (*GetArtifactsByContextResponse, error) // Gets all direct executions that a context associates with. GetExecutionsByContext(ctx context.Context, in *GetExecutionsByContextRequest, opts ...grpc.CallOption) (*GetExecutionsByContextResponse, error) + // TODO(b/283852485): Deprecate GetLineageGraph API after migration to + // GetLineageSubgraph API. // The transaction performs a constrained transitive closure and returns a // lineage subgraph satisfying the conditions and constraints specified in // the GetLineageGraphRequest. GetLineageGraph(ctx context.Context, in *GetLineageGraphRequest, opts ...grpc.CallOption) (*GetLineageGraphResponse, error) + // Gets a lineage subgraph by performing graph traversal from a list of + // interested nodes. + // A lineage subgraph without node details (e.g., external_id, properties) + // will be returned. + GetLineageSubgraph(ctx context.Context, in *GetLineageSubgraphRequest, opts ...grpc.CallOption) (*GetLineageSubgraphResponse, error) } type metadataStoreServiceClient struct { @@ -327,6 +404,15 @@ func (c *metadataStoreServiceClient) PutExecution(ctx context.Context, in *PutEx return out, nil } +func (c *metadataStoreServiceClient) PutLineageSubgraph(ctx context.Context, in *PutLineageSubgraphRequest, opts ...grpc.CallOption) (*PutLineageSubgraphResponse, error) { + out := new(PutLineageSubgraphResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/PutLineageSubgraph", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *metadataStoreServiceClient) PutContexts(ctx context.Context, in *PutContextsRequest, opts ...grpc.CallOption) (*PutContextsResponse, error) { out := new(PutContextsResponse) err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/PutContexts", in, out, opts...) @@ -570,6 +656,60 @@ func (c *metadataStoreServiceClient) GetEventsByArtifactIDs(ctx context.Context, return out, nil } +func (c *metadataStoreServiceClient) GetArtifactsByExternalIds(ctx context.Context, in *GetArtifactsByExternalIdsRequest, opts ...grpc.CallOption) (*GetArtifactsByExternalIdsResponse, error) { + out := new(GetArtifactsByExternalIdsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetArtifactsByExternalIds", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataStoreServiceClient) GetExecutionsByExternalIds(ctx context.Context, in *GetExecutionsByExternalIdsRequest, opts ...grpc.CallOption) (*GetExecutionsByExternalIdsResponse, error) { + out := new(GetExecutionsByExternalIdsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetExecutionsByExternalIds", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataStoreServiceClient) GetContextsByExternalIds(ctx context.Context, in *GetContextsByExternalIdsRequest, opts ...grpc.CallOption) (*GetContextsByExternalIdsResponse, error) { + out := new(GetContextsByExternalIdsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetContextsByExternalIds", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataStoreServiceClient) GetArtifactTypesByExternalIds(ctx context.Context, in *GetArtifactTypesByExternalIdsRequest, opts ...grpc.CallOption) (*GetArtifactTypesByExternalIdsResponse, error) { + out := new(GetArtifactTypesByExternalIdsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetArtifactTypesByExternalIds", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataStoreServiceClient) GetExecutionTypesByExternalIds(ctx context.Context, in *GetExecutionTypesByExternalIdsRequest, opts ...grpc.CallOption) (*GetExecutionTypesByExternalIdsResponse, error) { + out := new(GetExecutionTypesByExternalIdsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetExecutionTypesByExternalIds", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataStoreServiceClient) GetContextTypesByExternalIds(ctx context.Context, in *GetContextTypesByExternalIdsRequest, opts ...grpc.CallOption) (*GetContextTypesByExternalIdsResponse, error) { + out := new(GetContextTypesByExternalIdsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetContextTypesByExternalIds", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *metadataStoreServiceClient) GetContextsByArtifact(ctx context.Context, in *GetContextsByArtifactRequest, opts ...grpc.CallOption) (*GetContextsByArtifactResponse, error) { out := new(GetContextsByArtifactResponse) err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetContextsByArtifact", in, out, opts...) @@ -606,6 +746,24 @@ func (c *metadataStoreServiceClient) GetChildrenContextsByContext(ctx context.Co return out, nil } +func (c *metadataStoreServiceClient) GetParentContextsByContexts(ctx context.Context, in *GetParentContextsByContextsRequest, opts ...grpc.CallOption) (*GetParentContextsByContextsResponse, error) { + out := new(GetParentContextsByContextsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetParentContextsByContexts", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *metadataStoreServiceClient) GetChildrenContextsByContexts(ctx context.Context, in *GetChildrenContextsByContextsRequest, opts ...grpc.CallOption) (*GetChildrenContextsByContextsResponse, error) { + out := new(GetChildrenContextsByContextsResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetChildrenContextsByContexts", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *metadataStoreServiceClient) GetArtifactsByContext(ctx context.Context, in *GetArtifactsByContextRequest, opts ...grpc.CallOption) (*GetArtifactsByContextResponse, error) { out := new(GetArtifactsByContextResponse) err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetArtifactsByContext", in, out, opts...) @@ -633,6 +791,15 @@ func (c *metadataStoreServiceClient) GetLineageGraph(ctx context.Context, in *Ge return out, nil } +func (c *metadataStoreServiceClient) GetLineageSubgraph(ctx context.Context, in *GetLineageSubgraphRequest, opts ...grpc.CallOption) (*GetLineageSubgraphResponse, error) { + out := new(GetLineageSubgraphResponse) + err := c.cc.Invoke(ctx, "/ml_metadata.MetadataStoreService/GetLineageSubgraph", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // MetadataStoreServiceServer is the server API for MetadataStoreService service. // All implementations must embed UnimplementedMetadataStoreServiceServer // for forward compatibility @@ -702,6 +869,10 @@ type MetadataStoreServiceServer interface { // For new artifacts, type must be specified. // For old artifacts, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated artifacts will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // artifacts: A list of artifacts to insert or update. // @@ -715,6 +886,10 @@ type MetadataStoreServiceServer interface { // For new executions, type must be specified. // For old executions, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated executions will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // executions: A list of executions to insert or update. // @@ -726,6 +901,10 @@ type MetadataStoreServiceServer interface { // // The execution_id and artifact_id must already exist. // Once created, events cannot be modified. + // AlreadyExists error will be raised if duplicated events are found. + // + // It is not guaranteed that the created or updated events will share the + // same `milliseconds_since_epoch` timestamps. // // Args: // events: A list of events to insert or update. @@ -736,9 +915,16 @@ type MetadataStoreServiceServer interface { // input/output Event. The `contexts` describe the associations of the // execution and the attributions of the artifacts. // - // If an execution_id, artifact_id or context_id is specified, it is an - // update, otherwise it does an insertion. For insertion, type must be - // specified. + // If an execution_id is specified, it is an update on the corresponding + // execution, otherwise it does an insertion. + // For insertion, type must be specified. Same rule applies to artifacts + // and contexts in the request. Corresponding errors may raised. For example: + // AlreadyExists error will be raised if duplicated executions, artifacts + // or events are found. + // + // It is not guaranteed that the created or updated executions, artifacts, + // contexts and events will share the same `create_time_since_epoch`, + // `last_update_time_since_epoch`, or `milliseconds_since_epoch` timestamps. // // Args: // execution: An execution to insert or update. @@ -749,6 +935,36 @@ type MetadataStoreServiceServer interface { // An execution id and a list of artifacts and contexts ids index-aligned // with the input. PutExecution(context.Context, *PutExecutionRequest) (*PutExecutionResponse, error) + // Inserts or updates a lineage subgraph (i.e. a collection of event edges + // and its executions, artifacts, and related contexts) atomically. The + // `event_edges` include an Event and the indices of the corresponding + // execution and artifact from the input list of executions and artifacts. The + // `contexts` describe the associations of the Execution and the attributions + // of the Artifact. + // + // If an execution_id is specified, it is an update on the corresponding + // Execution, otherwise it does an insertion. For insertion, type must be + // specified. These rules apply to Artifacts and Contexts as well. + // Corresponding errors may be raised. For example: AlreadyExists error will + // be raised if duplicated executions, artifacts, or events are found. + // + // It is not guaranteed that the created or updated executions, artifacts, + // contexts and events will share the same `create_time_since_epoch`, + // `last_update_time_since_epoch`, or `milliseconds_since_epoch` timestamps. + // + // Args: + // executions: A list of executions to insert or update. + // artifacts: A list of artifacts to insert or update. + // contexts: A list of contexts to insert and/or create associations and + // attributions with. + // event_edges: A list of events to insert with the indices of the + // corresponding execution and artifact from the input lists of + // executions and artifacts. + // + // Returns: + // Lists of execution, artifact, and context ids index-aligned with the + // inputs. + PutLineageSubgraph(context.Context, *PutLineageSubgraphRequest) (*PutLineageSubgraphResponse, error) // Inserts or updates contexts in database and returns a list of context ids. // // If an context_id is specified for a context, it is an update. @@ -756,6 +972,10 @@ type MetadataStoreServiceServer interface { // For new contexts, type must be specified. // For old contexts, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated contexts will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // contexts: A list of contexts to insert or update. // @@ -848,6 +1068,18 @@ type MetadataStoreServiceServer interface { GetEventsByExecutionIDs(context.Context, *GetEventsByExecutionIDsRequest) (*GetEventsByExecutionIDsResponse, error) // Gets all events with matching artifact ids. GetEventsByArtifactIDs(context.Context, *GetEventsByArtifactIDsRequest) (*GetEventsByArtifactIDsResponse, error) + // Gets all the artifacts with matching external ids. + GetArtifactsByExternalIds(context.Context, *GetArtifactsByExternalIdsRequest) (*GetArtifactsByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetExecutionsByExternalIds(context.Context, *GetExecutionsByExternalIdsRequest) (*GetExecutionsByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetContextsByExternalIds(context.Context, *GetContextsByExternalIdsRequest) (*GetContextsByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetArtifactTypesByExternalIds(context.Context, *GetArtifactTypesByExternalIdsRequest) (*GetArtifactTypesByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetExecutionTypesByExternalIds(context.Context, *GetExecutionTypesByExternalIdsRequest) (*GetExecutionTypesByExternalIdsResponse, error) + // Gets all the artifacts with matching external ids. + GetContextTypesByExternalIds(context.Context, *GetContextTypesByExternalIdsRequest) (*GetContextTypesByExternalIdsResponse, error) // Gets all context that an artifact is attributed to. GetContextsByArtifact(context.Context, *GetContextsByArtifactRequest) (*GetContextsByArtifactResponse, error) // Gets all context that an execution is associated with. @@ -856,14 +1088,26 @@ type MetadataStoreServiceServer interface { GetParentContextsByContext(context.Context, *GetParentContextsByContextRequest) (*GetParentContextsByContextResponse, error) // Gets all children contexts that a context is related. GetChildrenContextsByContext(context.Context, *GetChildrenContextsByContextRequest) (*GetChildrenContextsByContextResponse, error) + // Batch getting all the parent contexts that a list of contexts are related. + GetParentContextsByContexts(context.Context, *GetParentContextsByContextsRequest) (*GetParentContextsByContextsResponse, error) + // Batch getting all the children contexts that a list of contexts are + // related. + GetChildrenContextsByContexts(context.Context, *GetChildrenContextsByContextsRequest) (*GetChildrenContextsByContextsResponse, error) // Gets all direct artifacts that a context attributes to. GetArtifactsByContext(context.Context, *GetArtifactsByContextRequest) (*GetArtifactsByContextResponse, error) // Gets all direct executions that a context associates with. GetExecutionsByContext(context.Context, *GetExecutionsByContextRequest) (*GetExecutionsByContextResponse, error) + // TODO(b/283852485): Deprecate GetLineageGraph API after migration to + // GetLineageSubgraph API. // The transaction performs a constrained transitive closure and returns a // lineage subgraph satisfying the conditions and constraints specified in // the GetLineageGraphRequest. GetLineageGraph(context.Context, *GetLineageGraphRequest) (*GetLineageGraphResponse, error) + // Gets a lineage subgraph by performing graph traversal from a list of + // interested nodes. + // A lineage subgraph without node details (e.g., external_id, properties) + // will be returned. + GetLineageSubgraph(context.Context, *GetLineageSubgraphRequest) (*GetLineageSubgraphResponse, error) mustEmbedUnimplementedMetadataStoreServiceServer() } @@ -895,6 +1139,9 @@ func (UnimplementedMetadataStoreServiceServer) PutEvents(context.Context, *PutEv func (UnimplementedMetadataStoreServiceServer) PutExecution(context.Context, *PutExecutionRequest) (*PutExecutionResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method PutExecution not implemented") } +func (UnimplementedMetadataStoreServiceServer) PutLineageSubgraph(context.Context, *PutLineageSubgraphRequest) (*PutLineageSubgraphResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PutLineageSubgraph not implemented") +} func (UnimplementedMetadataStoreServiceServer) PutContexts(context.Context, *PutContextsRequest) (*PutContextsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method PutContexts not implemented") } @@ -976,6 +1223,24 @@ func (UnimplementedMetadataStoreServiceServer) GetEventsByExecutionIDs(context.C func (UnimplementedMetadataStoreServiceServer) GetEventsByArtifactIDs(context.Context, *GetEventsByArtifactIDsRequest) (*GetEventsByArtifactIDsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetEventsByArtifactIDs not implemented") } +func (UnimplementedMetadataStoreServiceServer) GetArtifactsByExternalIds(context.Context, *GetArtifactsByExternalIdsRequest) (*GetArtifactsByExternalIdsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetArtifactsByExternalIds not implemented") +} +func (UnimplementedMetadataStoreServiceServer) GetExecutionsByExternalIds(context.Context, *GetExecutionsByExternalIdsRequest) (*GetExecutionsByExternalIdsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetExecutionsByExternalIds not implemented") +} +func (UnimplementedMetadataStoreServiceServer) GetContextsByExternalIds(context.Context, *GetContextsByExternalIdsRequest) (*GetContextsByExternalIdsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetContextsByExternalIds not implemented") +} +func (UnimplementedMetadataStoreServiceServer) GetArtifactTypesByExternalIds(context.Context, *GetArtifactTypesByExternalIdsRequest) (*GetArtifactTypesByExternalIdsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetArtifactTypesByExternalIds not implemented") +} +func (UnimplementedMetadataStoreServiceServer) GetExecutionTypesByExternalIds(context.Context, *GetExecutionTypesByExternalIdsRequest) (*GetExecutionTypesByExternalIdsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetExecutionTypesByExternalIds not implemented") +} +func (UnimplementedMetadataStoreServiceServer) GetContextTypesByExternalIds(context.Context, *GetContextTypesByExternalIdsRequest) (*GetContextTypesByExternalIdsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetContextTypesByExternalIds not implemented") +} func (UnimplementedMetadataStoreServiceServer) GetContextsByArtifact(context.Context, *GetContextsByArtifactRequest) (*GetContextsByArtifactResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetContextsByArtifact not implemented") } @@ -988,6 +1253,12 @@ func (UnimplementedMetadataStoreServiceServer) GetParentContextsByContext(contex func (UnimplementedMetadataStoreServiceServer) GetChildrenContextsByContext(context.Context, *GetChildrenContextsByContextRequest) (*GetChildrenContextsByContextResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetChildrenContextsByContext not implemented") } +func (UnimplementedMetadataStoreServiceServer) GetParentContextsByContexts(context.Context, *GetParentContextsByContextsRequest) (*GetParentContextsByContextsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetParentContextsByContexts not implemented") +} +func (UnimplementedMetadataStoreServiceServer) GetChildrenContextsByContexts(context.Context, *GetChildrenContextsByContextsRequest) (*GetChildrenContextsByContextsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetChildrenContextsByContexts not implemented") +} func (UnimplementedMetadataStoreServiceServer) GetArtifactsByContext(context.Context, *GetArtifactsByContextRequest) (*GetArtifactsByContextResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetArtifactsByContext not implemented") } @@ -997,6 +1268,9 @@ func (UnimplementedMetadataStoreServiceServer) GetExecutionsByContext(context.Co func (UnimplementedMetadataStoreServiceServer) GetLineageGraph(context.Context, *GetLineageGraphRequest) (*GetLineageGraphResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetLineageGraph not implemented") } +func (UnimplementedMetadataStoreServiceServer) GetLineageSubgraph(context.Context, *GetLineageSubgraphRequest) (*GetLineageSubgraphResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetLineageSubgraph not implemented") +} func (UnimplementedMetadataStoreServiceServer) mustEmbedUnimplementedMetadataStoreServiceServer() {} // UnsafeMetadataStoreServiceServer may be embedded to opt out of forward compatibility for this service. @@ -1154,6 +1428,24 @@ func _MetadataStoreService_PutExecution_Handler(srv interface{}, ctx context.Con return interceptor(ctx, in, info, handler) } +func _MetadataStoreService_PutLineageSubgraph_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PutLineageSubgraphRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).PutLineageSubgraph(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/PutLineageSubgraph", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).PutLineageSubgraph(ctx, req.(*PutLineageSubgraphRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _MetadataStoreService_PutContexts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PutContextsRequest) if err := dec(in); err != nil { @@ -1640,6 +1932,114 @@ func _MetadataStoreService_GetEventsByArtifactIDs_Handler(srv interface{}, ctx c return interceptor(ctx, in, info, handler) } +func _MetadataStoreService_GetArtifactsByExternalIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetArtifactsByExternalIdsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetArtifactsByExternalIds(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetArtifactsByExternalIds", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetArtifactsByExternalIds(ctx, req.(*GetArtifactsByExternalIdsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MetadataStoreService_GetExecutionsByExternalIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetExecutionsByExternalIdsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetExecutionsByExternalIds(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetExecutionsByExternalIds", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetExecutionsByExternalIds(ctx, req.(*GetExecutionsByExternalIdsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MetadataStoreService_GetContextsByExternalIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetContextsByExternalIdsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetContextsByExternalIds(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetContextsByExternalIds", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetContextsByExternalIds(ctx, req.(*GetContextsByExternalIdsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MetadataStoreService_GetArtifactTypesByExternalIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetArtifactTypesByExternalIdsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetArtifactTypesByExternalIds(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetArtifactTypesByExternalIds", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetArtifactTypesByExternalIds(ctx, req.(*GetArtifactTypesByExternalIdsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MetadataStoreService_GetExecutionTypesByExternalIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetExecutionTypesByExternalIdsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetExecutionTypesByExternalIds(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetExecutionTypesByExternalIds", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetExecutionTypesByExternalIds(ctx, req.(*GetExecutionTypesByExternalIdsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MetadataStoreService_GetContextTypesByExternalIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetContextTypesByExternalIdsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetContextTypesByExternalIds(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetContextTypesByExternalIds", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetContextTypesByExternalIds(ctx, req.(*GetContextTypesByExternalIdsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _MetadataStoreService_GetContextsByArtifact_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetContextsByArtifactRequest) if err := dec(in); err != nil { @@ -1712,6 +2112,42 @@ func _MetadataStoreService_GetChildrenContextsByContext_Handler(srv interface{}, return interceptor(ctx, in, info, handler) } +func _MetadataStoreService_GetParentContextsByContexts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetParentContextsByContextsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetParentContextsByContexts(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetParentContextsByContexts", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetParentContextsByContexts(ctx, req.(*GetParentContextsByContextsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _MetadataStoreService_GetChildrenContextsByContexts_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetChildrenContextsByContextsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetChildrenContextsByContexts(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetChildrenContextsByContexts", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetChildrenContextsByContexts(ctx, req.(*GetChildrenContextsByContextsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _MetadataStoreService_GetArtifactsByContext_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetArtifactsByContextRequest) if err := dec(in); err != nil { @@ -1766,6 +2202,24 @@ func _MetadataStoreService_GetLineageGraph_Handler(srv interface{}, ctx context. return interceptor(ctx, in, info, handler) } +func _MetadataStoreService_GetLineageSubgraph_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetLineageSubgraphRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataStoreServiceServer).GetLineageSubgraph(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/ml_metadata.MetadataStoreService/GetLineageSubgraph", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataStoreServiceServer).GetLineageSubgraph(ctx, req.(*GetLineageSubgraphRequest)) + } + return interceptor(ctx, in, info, handler) +} + // MetadataStoreService_ServiceDesc is the grpc.ServiceDesc for MetadataStoreService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -1805,6 +2259,10 @@ var MetadataStoreService_ServiceDesc = grpc.ServiceDesc{ MethodName: "PutExecution", Handler: _MetadataStoreService_PutExecution_Handler, }, + { + MethodName: "PutLineageSubgraph", + Handler: _MetadataStoreService_PutLineageSubgraph_Handler, + }, { MethodName: "PutContexts", Handler: _MetadataStoreService_PutContexts_Handler, @@ -1913,6 +2371,30 @@ var MetadataStoreService_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetEventsByArtifactIDs", Handler: _MetadataStoreService_GetEventsByArtifactIDs_Handler, }, + { + MethodName: "GetArtifactsByExternalIds", + Handler: _MetadataStoreService_GetArtifactsByExternalIds_Handler, + }, + { + MethodName: "GetExecutionsByExternalIds", + Handler: _MetadataStoreService_GetExecutionsByExternalIds_Handler, + }, + { + MethodName: "GetContextsByExternalIds", + Handler: _MetadataStoreService_GetContextsByExternalIds_Handler, + }, + { + MethodName: "GetArtifactTypesByExternalIds", + Handler: _MetadataStoreService_GetArtifactTypesByExternalIds_Handler, + }, + { + MethodName: "GetExecutionTypesByExternalIds", + Handler: _MetadataStoreService_GetExecutionTypesByExternalIds_Handler, + }, + { + MethodName: "GetContextTypesByExternalIds", + Handler: _MetadataStoreService_GetContextTypesByExternalIds_Handler, + }, { MethodName: "GetContextsByArtifact", Handler: _MetadataStoreService_GetContextsByArtifact_Handler, @@ -1929,6 +2411,14 @@ var MetadataStoreService_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetChildrenContextsByContext", Handler: _MetadataStoreService_GetChildrenContextsByContext_Handler, }, + { + MethodName: "GetParentContextsByContexts", + Handler: _MetadataStoreService_GetParentContextsByContexts_Handler, + }, + { + MethodName: "GetChildrenContextsByContexts", + Handler: _MetadataStoreService_GetChildrenContextsByContexts_Handler, + }, { MethodName: "GetArtifactsByContext", Handler: _MetadataStoreService_GetArtifactsByContext_Handler, @@ -1941,6 +2431,10 @@ var MetadataStoreService_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetLineageGraph", Handler: _MetadataStoreService_GetLineageGraph_Handler, }, + { + MethodName: "GetLineageSubgraph", + Handler: _MetadataStoreService_GetLineageSubgraph_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "ml_metadata/proto/metadata_store_service.proto", diff --git a/third_party/ml-metadata/ml_metadata/proto/metadata_store.proto b/third_party/ml-metadata/ml_metadata/proto/metadata_store.proto index b015dd22b1..3bdeae8791 100644 --- a/third_party/ml-metadata/ml_metadata/proto/metadata_store.proto +++ b/third_party/ml-metadata/ml_metadata/proto/metadata_store.proto @@ -16,6 +16,7 @@ syntax = "proto2"; package ml_metadata; +import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/descriptor.proto"; @@ -36,11 +37,13 @@ message Value { double double_value = 2; string string_value = 3; google.protobuf.Struct struct_value = 4; + google.protobuf.Any proto_value = 5; + bool bool_value = 6; } } message Artifact { - // Output only. The globally unique server generated id of the artifact. + // Output only. The unique server generated id of the artifact. optional int64 id = 1; // The client provided name of the artifact. This field is optional. If set, // it must be unique among all the artifacts of the same artifact type within @@ -54,6 +57,9 @@ message Artifact { // The uniform resource identifier of the physical artifact. // May be empty if there is no physical artifact. optional string uri = 3; + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all artifacts within a database instance. + optional string external_id = 11; // Properties of the artifact. // Properties must be specified in the ArtifactType. map properties = 4; @@ -71,6 +77,17 @@ message Artifact { MARKED_FOR_DELETION = 3; // A state indicating that the artifact has been deleted. DELETED = 4; + // A state indicating that the artifact has been abandoned, which may be + // due to a failed or cancelled execution. + ABANDONED = 5; + // A state indicating that the artifact is a reference artifact. At + // execution start time, the orchestrator produces an output artifact for + // each output key with state PENDING. However, for an intermediate + // artifact, this first artifact's state will be REFERENCE. Intermediate + // artifacts emitted during a component's execution will copy the REFERENCE + // artifact's attributes. At the end of an execution, the artifact state + // should remain REFERENCE instead of being changed to LIVE. + REFERENCE = 6; } // The state of the artifact known to the system. @@ -80,6 +97,9 @@ message Artifact { // Output only. Last update time of the artifact since epoch in millisecond // since epoch. optional int64 last_update_time_since_epoch = 10; + + // Output only. + optional google.protobuf.Any system_metadata = 12; } // The list of supported property value types. @@ -88,7 +108,11 @@ enum PropertyType { INT = 1; DOUBLE = 2; STRING = 3; + // Prefer to use `PROTO` to store structed data since this option has + // inefficient database storage usage. STRUCT = 4; + PROTO = 5; + BOOLEAN = 6; } message ArtifactType { @@ -101,6 +125,10 @@ message ArtifactType { optional string version = 4; // An optional description about the type. optional string description = 5; + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all artifact types within a database + // instance. + optional string external_id = 7; // The schema of the type. // Properties are always optional in the artifact. // Properties of an artifact type can be expanded but not contracted (i.e., @@ -156,40 +184,57 @@ message ArtifactType { // type:DECLARED_OUTPUT, // path:{step:[{"key":"my_result"}]} // } -// Other event types include INPUT/OUTPUT and INTERNAL_INPUT/_OUTPUT. +// +// Other event types include INPUT/OUTPUT, INTERNAL_INPUT/_OUTPUT and +// PENDING_OUTPUT: +// // * The INPUT/OUTPUT is an event that actually reads/writes an artifact by an // execution. The input/output artifacts may not declared in the signature, // For example, the trainer may output multiple caches of the parameters // (as an OUTPUT), then finally write the SavedModel as a DECLARED_OUTPUT. +// // * The INTERNAL_INPUT/_OUTPUT are event types which are only meaningful to // an orchestration system to keep track of the details for later debugging. // For example, a fork happened conditioning on an artifact, then an execution -// is triggered, such fork implementating may need to log the read and write +// is triggered, such fork implementing may need to log the read and write // of artifacts and may not be worth displaying to the users. // -// For instance, in the above example, +// For instance, in the above example, // -// my_result = my_execution({"data":[3,7],"schema":8}) +// my_result = my_execution({"data":[3,7],"schema":8}) // -// there is another execution (id: 15), which represents a `garbage_collection` -// step in an orchestration system +// there is another execution (id: 15), which represents a +// `garbage_collection` step in an orchestration system // -// gc_result = garbage_collection(my_result) +// gc_result = garbage_collection(my_result) // -// that cleans `my_result` if needed. The details should be invisible to the -// end users and lineage tracking. The orchestrator can emit following events: +// that cleans `my_result` if needed. The details should be invisible to the +// end users and lineage tracking. The orchestrator can emit following events: // -// { -// artifact_id: 15, -// execution_id: 15, -// type:INTERNAL_INPUT, -// } -// { -// artifact_id:16, // New artifact containing the GC job result. -// execution_id: 15, -// type:INTERNAL_OUTPUT, -// path:{step:[{"key":"gc_result"}]} -// } +// { +// artifact_id: 15, +// execution_id: 15, +// type:INTERNAL_INPUT, +// } +// { +// artifact_id:16, // New artifact containing the GC job result. +// execution_id: 15, +// type:INTERNAL_OUTPUT, +// path:{step:[{"key":"gc_result"}]} +// } +// +// * The PENDING_OUTPUT event is used to indicate that an artifact is +// tentatively associated with an active execution which has not yet been +// finalized. For example, an orchestration system can register output +// artifacts of a running execution with PENDING_OUTPUT events to indicate +// the output artifacts the execution is expected to produce. When the +// execution is finished, the final set of output artifacts can be associated +// with the exeution using OUTPUT events, and any unused artifacts which were +// previously registered with PENDING_OUTPUT events can be updated to set +// their Artifact.State to ABANDONED. +// +// Events are unique of the same +// (artifact_id, execution_id, type) combination within a metadata store. message Event { // A simple path (e.g. {step{key:"foo"}}) can name an artifact in the context // of an execution. @@ -216,6 +261,7 @@ message Event { OUTPUT = 4; // An output of the execution. INTERNAL_INPUT = 5; // An internal input of the execution. INTERNAL_OUTPUT = 6; // An internal output of the execution. + PENDING_OUTPUT = 7; // A pending output of the execution. } // The artifact id is required for an event, and should refer to an // existing artifact. @@ -230,10 +276,13 @@ message Event { // Time the event occurred // Epoch is Jan 1, 1970, UTC optional int64 milliseconds_since_epoch = 5; + + // Output only. + optional google.protobuf.Any system_metadata = 6; } message Execution { - // Output only. The globally unique server generated id of the execution. + // Output only. The unique server generated id of the execution. optional int64 id = 1; // The client provided name of the execution. This field is optional. If set, // it must be unique among all the executions of the same execution type @@ -245,6 +294,9 @@ message Execution { optional int64 type_id = 2; // Output only. The name of an ExecutionType. optional string type = 7; + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all executions within a database instance. + optional string external_id = 10; // The state of the Execution. The state transitions are // NEW -> RUNNING -> COMPLETE | CACHED | FAILED | CANCELED // CACHED means the execution is skipped due to cached results. @@ -272,6 +324,9 @@ message Execution { optional int64 create_time_since_epoch = 8; // Output only. Last update time of the execution in millisecond since epoch. optional int64 last_update_time_since_epoch = 9; + + // Output only. + optional google.protobuf.Any system_metadata = 11; } message ExecutionType { @@ -284,6 +339,10 @@ message ExecutionType { optional string version = 6; // An optional description about the type. optional string description = 7; + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all execution types within a database + // instance. + optional string external_id = 9; // The schema of the type. // Properties are always optional in the execution. map properties = 3; @@ -335,6 +394,10 @@ message ContextType { optional string version = 4; // An optional description about the type. optional string description = 5; + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all context types within a database + // instance. + optional string external_id = 7; // The schema of the type, e.g., name: string, owner: string // Properties are always optional in the context. // Properties of an context type can be expanded but not contracted (i.e., @@ -354,7 +417,7 @@ message ContextType { } message Context { - // Output Only. The globally unique server generated id of the context. + // Output Only. The unique server generated id of the context. optional int64 id = 1; // The client provided name of the context. It must be unique within a // database instance. @@ -364,6 +427,9 @@ message Context { optional int64 type_id = 2; // Output only. The name of a ContextType. optional string type = 6; + // The external id that come from the clients’ system. This field is optional. + // If set, it must be unique among all contexts within a virtual database. + optional string external_id = 9; // Values of the properties, which must be specified in the ContextType. map properties = 4; // User provided custom properties which are not defined by its type. @@ -372,6 +438,9 @@ message Context { optional int64 create_time_since_epoch = 7; // Output only. Last update time of the context in millisecond since epoch. optional int64 last_update_time_since_epoch = 8; + + // Output only system metadata. + optional google.protobuf.Any system_metadata = 10; } // the Attribution edges between Context and Artifact instances. @@ -534,7 +603,7 @@ message MySQLDatabaseConfig { optional string host = 1; // The TCP Port number that the MYSQL server accepts connections on. // If unspecified, the default MYSQL port (3306) is used. - optional uint32 port = 2; + optional int64 port = 2; // The database to connect to. Must be specified. // After connecting to the MYSQL server, this database is created if not // already present unless skip_db_creation is set. @@ -609,6 +678,71 @@ message SqliteMetadataSourceConfig { optional ConnectionMode connection_mode = 2; } +// A config contains the parameters when using with PostgreSQLMetadatSource. +// Next index: 10 +message PostgreSQLDatabaseConfig { + // Name of host to connect to. If the host name starts with /, it is taken as + // a Unix-domain socket in the abstract namespace. + optional string host = 1; + + // Numeric IP address of host to connect to. If this field is provided, `host` + // field is ignored. + optional string hostaddr = 2; + + // Port number to connect to at the server host, or socket file name extension + // for Unix-domain connections. + optional string port = 3; + + // PostgreSQL user name to connect as. Defaults to be the same as the + // operating system name of the user running the application. + optional string user = 4; + + // Password to be used if the server demands password authentication. + optional string password = 5; + + // Specifies the name of the file used to store passwords. + optional string passfile = 6; + + // The database name. Defaults to be the same as the user name. + optional string dbname = 7; + + // A config to skip the database creation if not exist when connecting the + // db instance. It is useful when the db creation is handled by an admin + // process, while the lib user should not issue db creation clauses. + optional bool skip_db_creation = 8; + + message SSLOptions { + // disable, allow, verify-ca, verify-full, etc. Reference: + // https://www.postgresql.org/docs/current/libpq-connect.html + optional string sslmode = 1; + + // This parameter specifies the file name of the client SSL certificate, + // replacing the default ~/.postgresql/postgresql.crt. This parameter is + // ignored if an SSL connection is not made. + optional string sslcert = 2; + + // This parameter specifies the location for the secret key used for the + // client certificate. It can either specify a file name that will be used + // instead of the default ~/.postgresql/postgresql.key, this parameter is + // ignored if an SSL connection is not made. + optional string sslkey = 3; + + // This parameter specifies the password for the secret key specified in + // sslkey, allowing client certificate private keys to be stored in + // encrypted form on disk even when interactive passphrase input is not + // practical. + optional string sslpassword = 4; + + // This parameter specifies the name of a file containing SSL certificate + // authority (CA) certificate(s). If the file exists, the server's + // certificate will be verified to be signed by one of these authorities. + // The default is ~/.postgresql/root.crt. + optional string sslrootcert = 5; + } + + optional SSLOptions ssloption = 9; +} + message MigrationOptions { // If not set, by default the upgrade migration is disabled. MLMD only @@ -647,6 +781,10 @@ message ConnectionConfig { FakeDatabaseConfig fake_database = 1; MySQLDatabaseConfig mysql = 2; SqliteMetadataSourceConfig sqlite = 3; + + // PostgreSQL database connection config. + PostgreSQLDatabaseConfig postgresql = 5; + } // Options for overwriting the default retry setting when MLMD transactions @@ -672,7 +810,7 @@ message MetadataStoreClientConfig { optional string host = 1; // The TCP Port number that the gRPC server accepts connections on. // Must be specified. - optional uint32 port = 2; + optional int64 port = 2; message SSLConfig { // The PEM-encoded private key as a byte string, or Empty if no private key @@ -757,12 +895,14 @@ message ListOperationOptions { optional string next_page_token = 3; // A boolean expression in SQL syntax that is used to specify the conditions - // on nodes' attributes and 1-hop neighborhood. + // on node attributes and directly connected assets. // // In the current implementation, filtering Artifact/Execution/Context with // the following attributes and neighborhood is supported: // - // Attributes: id:int64, type_id:int64, type:string, uri:string, name: string, + // Attributes: + // id:int64, type_id:int64, type:string, + // uri:string, name: string, external_id: string, // create_time_since_epoch:int64, last_update_time_since_epoch:int64 // state:ENUM (Artifact only) last_known_state:ENUM (Execution only) // @@ -772,6 +912,7 @@ message ListOperationOptions { // custom_properties.$name ($name is the custom property name) // attributes: the following attributes can be used // int_value: int64, double_value: double, string_value: string + // bool_value: bool // // - Context (for Artifact and Execution): // syntax: contexts_$alias ($alias can be [0-9A-Za-z_]) @@ -799,6 +940,7 @@ message ListOperationOptions { // - type = 'my_type_name' // - name = 'foo' // - type = 'bar' AND name LIKE 'foo%' + // - external_id = 'my_external_id' // - NOT(create_time_since_epoch < 1 OR last_update_time_since_epoch < 1) // // b) to filter artifacts' uri @@ -809,10 +951,12 @@ message ListOperationOptions { // c) to filter artifact's state or execution's last_known_state // - state = LIVE // - state IS NULL + // - state IN (PENDING, LIVE) // - last_known_state = RUNNING // - last_known_state != RUNNING + // - last_known_state NOT IN (FAILED, CANCELED) // - // d) to filter nodes having a specific context + // d) to filter nodes having a specific context, artifact, or execution // - contexts_a.id = 5 // - contexts_a.type = 'RunContext' // - contexts_a.name = 'my_run' @@ -820,6 +964,29 @@ message ListOperationOptions { // - contexts_a.last_update_time_since_epoch = 1626761453 // To filter nodes with conditions on multiple contexts: // - contexts_a.name = 'my_run' AND contexts_b.name = 'my_pipeline' + // To filter context with artifacts: + // - artifacts_a.id = 5 + // - artifacts_a.type = 'Dataset' + // - artifacts_a.name = 'my_dataset' + // - artifacts_a.uri = 'exact_path_string' + // - artifacts_a.state = LIVE + // - artifacts_a.state IN (PENDING, LIVE) + // - artifacts_a.external_id = "my_external_id" + // - artifacts_a.create_time_since_epoch = 1626761453 + // - artifacts_a.last_update_time_since_epoch = 1626761453 + // To filter contexts with conditions on multiple artifacts: + // - artifacts_a.name = 'my_run' AND artifacts_b.name = 'my_pipeline' + // To filter context with executions: + // - executions_a.id = 5 + // - executions_a.type = 'Dataset' + // - executions_a.name = 'my_dataset' + // - executions_a.last_known_state = RUNNING + //. - executions_a.last_known_state IN (NEW, RUNNING) + // - executions_a.external_id = "my_external_id" + // - executions_a.create_time_since_epoch = 1626761453 + // - executions_a.last_update_time_since_epoch = 1626761453 + // To filter contexts with conditions on multiple executions: + // - executions_a.name = 'my_run' AND executions_b.name = 'my_pipeline' // // e) to filter nodes condition on their properties // - properties.accuracy.double_value > 0.95 @@ -828,7 +995,7 @@ message ListOperationOptions { // other than [0-9A-Za-z_], then the name need to be backquoted, // e.g., // - properties.`my property`.int_value > 0 - // - custom_properties.`my:custom.property`.string_value = 'foo' + // - custom_properties.`my:custom.property`.bool_value = true // // f) complex query to filter both node attributes and neighborhood // - type = 'DataSet' AND @@ -846,7 +1013,7 @@ message ListOperationOptions { // - events_0.milliseconds_since_epoch = 1 // to filter Executions on Events // - events_0.artifact_id = 1 - // - events_0.type = INPUT + // - events_0.type IN (INPUT, INTERNAL_INPUT) // - events_0.milliseconds_since_epoch = 1 // TODO(b/145945460) Support filtering on event step fields. optional string filter_query = 4; @@ -877,48 +1044,190 @@ message ListOperationNextPageToken { // the same order_by field value. // This field is currently only set whe order_by field is LAST_UPDATE_TIME. repeated int64 listed_ids = 4; + } // Options for transactions. // Note: This is under development. Clients should not use it. message TransactionOptions { extensions 1000 to max; + + // Transaction tag for debug use only. + optional string tag = 1; } -// The query options for list lineage graph operation. It allows specifying the -// `query_nodes` of interests and the `stop_conditions` when querying a -// lineage graph. The query option is used for exporting provenance information -// from a source MLMD instance. +// TODO(b/283852485): Deprecate GetLineageGraph API after migration to +// GetLineageSubgraph API. +// The query options for `get_lineage_graph` operation. +// `query_nodes` is a list of nodes of interest. +// Currently only artifacts are supported as `query_nodes`. +// `stop_conditions` defines the filtering rules when querying a lineage graph. +// `max_node_size` defines the total number of artifacts and executions returned +// in the subgraph. message LineageGraphQueryOptions { - // A query to specify the nodes of interests. + // A query to specify the nodes of interest. + // `ListOperationOptions.max_result_size` sets the maximum number of nodes to + // begin with the graph search. // TODO(b/178491112) Support query_nodes for Executions. oneof query_nodes { ListOperationOptions artifacts_options = 1; } - // Boundary conditions to stop the traversal when return the `subgraph`. + // Filtering conditions for retrieving the lineage graph. message BoundaryConstraint { - // The maximum number of hops from the `query_nodes` to traverse. It should - // be non-negative. When zero, only the `query_nodes` are returned. + // The maximum number of hops from the `query_nodes` to traverse. + // A hop is defined as a jump to the next node following the path of + // node -> event -> next_node. + // For example, in the lineage graph a_1 -> e_1 -> a_2: + // a_2 is 2 hops away from a_1, and e_1 is 1 hop away from a_1. + // `max_num_hops` should be non-negative. + // When its value is set to 0, only the `query_nodes` are returned. optional int64 max_num_hops = 1; - // Conditions for the boundary node in the returned the subgraph. - // Please refer to ListOperationOptions.filter_query for the syntax. - // If set, it stops traversing further at the artifacts that do not satisfy - // `boundary_artifacts` and exclude them from the returned subgraph. - // For example, to look for the models related to a DataSet and ignore - // other artifacts derived from the DataSet: - // "type = 'DataSet' OR type = 'TransformGraph' OR type = 'Model'" + // Filtering conditions for retrieving the lineage graph. + // Please refer to `ListOperationOptions.filter_query` for the syntax. + // + // If set, the `boundary_artifacts` defines which artifacts to keep in the + // returned lineage graph during the graph search. + // Artifacts that do not satisfy the `boundary_artifacts` are filtered out, + // and the subgraphs starting at them will be pruned. + // If not set, no artifacts will be filtered out. + // Taking the following lineage graph as example: + // (`a` represents an Artifact, `e` represents an Execution, each arrow + // represents a hop.) + // a_0 a_1 a_3 + // | \ / \ + // \/ \/ \/ \/ + // e_0 e_1 e_3 + // / \ + // \/ \/ + // a_2 a_4 a_5 + // \ / + // \/ \/ + // e_2 + // To query all the upstream and downstream nodes 3 hops away from a_4, + // while excluding the upstream subgraph starting at a_3, then + // `stop_conditions` can be set as: + // { + // max_num_hops: 3 + // boundary_artifacts: 'id != 3' + // } + // With the `stop_conditions`, {a_3, e_1, a_1, a_0, e_0} will be filtered + // out. + // The returned lineage graph looks like: + // e_3 + // / \ + // \/ \/ + // a_2 a_4 a_5 + // \ / + // \/ \/ + // e_2 optional string boundary_artifacts = 2; - // If set, it stops traversing further at the executions that do not satisfy - // `boundary_executions` and exclude them from the returned subgraph. - // For example, two trainers may be connected due to the shared datasets, - // to exclude other trainers except a particular one (e.g., id = 5): - // "type != 'Trainer' OR (type = 'Trainer' AND id = 5)" + // If set, the `boundary_executions` defines which executions to keep in the + // returned lineage graph during the graph search. + // Executions that do not satisfy the `boundary_executions` are filtered out + // and the subgraphs starting at them will be pruned. + // If not set, no executions will be filtered out. + // In the example above, to query for all the upstream and downstream nodes + // 3 hops away from a_4, while excluding the upstream subgraph and the + // downstream subgraph starting at e_3, then `stop_conditions` can be set as + // { + // max_num_hops: 3 + // boundary_executions: 'id != 3' + // } + // With the `stop_conditions`, {e_3, a_5, a_3, e_1, a_1, a_0, e_0} will be + // filtered out. + // The returned lineage graph looks like: + // a_2 a_4 + // \ / + // \/ \/ + // e_2 + // However, for the following graph: + // a_0 a_1 a_3 + // | \ / \ + // \/ \/ \/ \/ + // e_0 e_1 e_3 + // \ / \ + // \/ \/ \/ + // a_2 a_4 a_5 + // \ / + // \/ \/ + // e_2 + // With the same `stop_conditions`, only {e_3, a_5, a_0, e_0} will be + // filtered out. + // The returned lineage graph looks like: + // a_1 a_3 + // \ / + // \/ \/ + // e_1 + // \ + // \/ + // a_2 a_4 + // \ / + // \/ \/ + // e_2 optional string boundary_executions = 3; } - // A constraint option to define the boundary of the returned subgraph. + // A constraint option to define the filtering rules when querying a lineage + // graph. optional BoundaryConstraint stop_conditions = 2; + + // Maximum total number of artifacts and executions in the whole returned + // lineage graph. + // If set to 0 or below, all related nodes will be returned without any + // number limitation. + // The number counts toward Artifacts and Executions. Nothing else considered. + // + // NOTE: There is no pagination supported. + optional int64 max_node_size = 3 [default = 20]; +} + +// The query options for lineage graph tracing from a list of interested nodes. +message LineageSubgraphQueryOptions { + // `starting_nodes` is a list of nodes of interest to start graph tracing. + // NOTE: The maximum number of starting nodes is 100 at most. + message StartingNodes { + // `filter_query` is a boolean expression in SQL syntax that is used to + // specify the conditions on starting nodes. + // Please refer to ListOperationOptions.filter_query for more details. + optional string filter_query = 1; + } + + oneof starting_nodes { + StartingNodes starting_artifacts = 1; + StartingNodes starting_executions = 2; + } + + // The maximum number of hops from the `starting_nodes` to traverse. + // A hop is defined as a jump to the next node following the path of + // node -> event -> next_node. + // For example, in the lineage graph a_1 -> e_1 -> a_2: + // a_2 is 2 hops away from a_1, and e_1 is 1 hop away from a_1. + // `max_num_hops` should be non-negative. + // When its value is set to 0, only the `starting_nodes` are returned. + optional int64 max_num_hops = 3; + + enum Direction { + // Direction is by defult DIRECTION_UNSPECIFIED, which is equivalent to + // BIDIRECTIONAL. + DIRECTION_UNSPECIFIED = 0; + // Indicates tracing the lineage graph by hops in upstream direction. + UPSTREAM = 1; + // Indicates tracing the lineage graph by hops in downstream direction. + DOWNSTREAM = 2; + // Indicates tracing the lineage graph in both directions. + BIDIRECTIONAL = 3; + } + // The direction of lineage graph tracing, which means the direction of all + // hops in the tracing. + // An UPSTREAM hop means an expansion following the path of + // execution -> output_event -> artifact or + // artifact -> input_event -> execution + // A DOWNSTREAM hop means an expansion following the path of + // execution -> input_event -> artifact or + // artifact -> output_event -> execution + // Please refer to `Direction` for more details. + optional Direction direction = 4; } diff --git a/third_party/ml-metadata/ml_metadata/proto/metadata_store_service.proto b/third_party/ml-metadata/ml_metadata/proto/metadata_store_service.proto index a1155eed78..773a968ab9 100644 --- a/third_party/ml-metadata/ml_metadata/proto/metadata_store_service.proto +++ b/third_party/ml-metadata/ml_metadata/proto/metadata_store_service.proto @@ -17,6 +17,7 @@ syntax = "proto2"; package ml_metadata; +import "google/protobuf/field_mask.proto"; import "ml_metadata/proto/metadata_store.proto"; // An artifact and type pair. Part of an artifact struct. @@ -69,6 +70,94 @@ message PutArtifactsRequest { // Additional options to change the behavior of the method. optional Options options = 2; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 3; + + // FieldMask for artifacts in the PUT update + // If `artifact.id` is not specified, it means a new artifact will be created + // and `update_mask` will not be applied to the creation. + // If `update_mask` is empty, update the artifacts as a whole. + // If `update_mask` is not empty, only update fields or properties specified + // in `update_mask`. + // Example request protos: + // 1. Examples that update `properties` / `custom_properties`: + // 1.1 Add a <'key', 'val'> pair into `custom_properties`: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // string_value: "val" + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 1.2 Set `custom_properties['key'].bool_value` to true: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // bool_value: true + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 1.3 Delete the complete <'key', 'val'> pair from `custom_properties`: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // custom_properties {} + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 2. Examples that update fields such as `uri`, `external_id`, etc: + // 2.1 Update `external_id` field: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // external_id: "new_value" + // } + // update_mask { + // paths: "external_id" + // } + // } + // 2.2 Set `uri` field: + // { + // artifacts { + // id: 1234 + // type_id: 5678 + // uri: "set_value" + // } + // update_mask { + // paths: "uri" + // } + // } + // If `paths: "properties"` or `paths: "custom_properties"` are added to + // `update_mask`, the key-level updates will be ignored and we only perform + // field-level updates on the all `properties`/`custom_properties`. + // For example: + // If the mask is: {"properties", "properties.key1"}, the field path + // "properties.key1" will be ignored and all `properties` will be updated. + // (Do not suggest) + // If the mask is {"properties", "external_id"}, all + // `properties` and field `external_id` will be updated. (Do not suggest) + optional google.protobuf.FieldMask update_mask = 4; } message PutArtifactsResponse { @@ -97,6 +186,9 @@ message PutArtifactTypeRequest { // Deprecated fields. optional bool can_delete_fields = 3 [deprecated = true]; optional bool all_fields_match = 4 [default = true, deprecated = true]; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 6; } message PutArtifactTypeResponse { @@ -106,6 +198,46 @@ message PutArtifactTypeResponse { message PutExecutionsRequest { repeated Execution executions = 1; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; + + // FieldMask for executions in the PUT update + // If `execution.id` is not specified, it means a new execution will be + // created and `update_mask` will not be applied to the creation. + // If `update_mask` is empty, update the executions as a whole. + // If `update_mask` is not empty, only update fields or properties specified + // in `update_mask`. + // Example request protos: + // 1. Add a <'key', 'val'> pair into `custom_properties`: + // { + // executions { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // string_value: "val" + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 2. Set `last_known_state` field: + // { + // executions { + // id: 1234 + // type_id: 5678 + // last_known_state: CACHED + // } + // update_mask { + // paths: "last_known_state" + // } + // } + // Please refer to `PutArtifactsRequest` for more details. + optional google.protobuf.FieldMask update_mask = 3; } message PutExecutionsResponse { @@ -134,6 +266,9 @@ message PutExecutionTypeRequest { // Deprecated fields. optional bool can_delete_fields = 3 [deprecated = true]; optional bool all_fields_match = 4 [default = true, deprecated = true]; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 6; } message PutExecutionTypeResponse { @@ -143,6 +278,9 @@ message PutExecutionTypeResponse { message PutEventsRequest { repeated Event events = 1; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; } message PutEventsResponse {} @@ -165,9 +303,19 @@ message PutExecutionRequest { message Options { // When there's a race to publish executions with a new context with the // same context.name, by default there'll be one writer succeeds and - // the rest of the writers returning AlreadyExists errors. If set the field, - // the failed writer will reuse the stored context in the transaction. + // the rest of the writers returning AlreadyExists errors. If set to true, + // the API will reuse the stored context in the transaction and perform + // an update. optional bool reuse_context_if_already_exist = 1; + + // When there's a race to publish executions with a new artifact with the + // same artifact.external_id, by default there'll be one writer succeeds and + // the rest of the writers returning AlreadyExists errors. + // If set to true and an Artifact has non-empty external_id, + // the API will reuse the stored artifact in the transaction and + // perform an update. Otherwise, it will fall back to relying on `id` field + // to decide if it's update (if `id` exists) or insert (if `id` is empty). + optional bool reuse_artifact_if_already_exist_by_external_id = 2; } // The execution that produces many artifact and event pairs. optional Execution execution = 1; @@ -182,6 +330,9 @@ message PutExecutionRequest { repeated Context contexts = 3; // Additional options to change the behavior of the method. optional Options options = 4; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 5; } message PutExecutionResponse { @@ -195,6 +346,51 @@ message PutExecutionResponse { repeated int64 context_ids = 3; } +message PutLineageSubgraphRequest { + repeated Execution executions = 1; + repeated Artifact artifacts = 2; + repeated Context contexts = 3; + + message EventEdge { + // Index in the array of executions. + optional int32 execution_index = 1; + // Index in the array of artifacts. + optional int32 artifact_index = 2; + optional Event event = 3; + } + repeated EventEdge event_edges = 4; + + message Options { + // When there's a race to publish executions with a new context with the + // same context.name, by default there'll be one writer succeeds and + // the rest of the writers returning AlreadyExists errors. If set to true, + // the API will reuse the stored context in the transaction and perform + // an update. + optional bool reuse_context_if_already_exist = 1; + + // When there's a race to publish executions with a new artifact with the + // same artifact.external_id, by default there'll be one writer succeeds and + // the rest of the writers returning AlreadyExists errors. + // If set to true and an Artifact has non-empty external_id, + // the API will reuse the stored artifact in the transaction and + // perform an update. Otherwise, it will fall back to relying on `id` field + // to decide if it's update (if `id` exists) or insert (if `id` is empty). + optional bool reuse_artifact_if_already_exist_by_external_id = 2; + } + optional Options options = 5; + + optional TransactionOptions transaction_options = 6; +} + +message PutLineageSubgraphResponse { + // A list of execution ids index-aligned with `executions` in the request + repeated int64 execution_ids = 1 [packed = true]; + // A list of artifact ids index-aligned with `artifacts` in the request + repeated int64 artifact_ids = 2 [packed = true]; + // A list of context ids index-aligned with `contexts` in the request + repeated int64 context_ids = 3 [packed = true]; +} + message PutTypesRequest { repeated ArtifactType artifact_types = 1; repeated ExecutionType execution_types = 2; @@ -214,6 +410,9 @@ message PutTypesRequest { // Deprecated fields. optional bool can_delete_fields = 5 [deprecated = true]; optional bool all_fields_match = 6 [default = true, deprecated = true]; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 8; } message PutTypesResponse { @@ -246,6 +445,9 @@ message PutContextTypeRequest { // Deprecated fields. optional bool can_delete_fields = 3 [deprecated = true]; optional bool all_fields_match = 4 [default = true, deprecated = true]; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 6; } message PutContextTypeResponse { @@ -255,6 +457,46 @@ message PutContextTypeResponse { message PutContextsRequest { repeated Context contexts = 1; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; + + // FieldMask for contexts in the PUT update + // If `context.id` is not specified, it means a new context will be + // created and `update_mask` will not be applied to the creation. + // If `update_mask` is empty, update the contexts as a whole. + // If `update_mask` is not empty, only update fields or properties specified + // in `update_mask`. + // Example request protos: + // 1. Add a <'key', 'val'> pair into `custom_properties`: + // { + // contexts { + // id: 1234 + // type_id: 5678 + // custom_properties { + // key: "key" + // value: { + // string_value: "val" + // } + // } + // } + // update_mask { + // paths: "custom_properties.key" + // } + // } + // 2. Set `name` field: + // { + // contexts { + // id: 1234 + // type_id: 5678 + // name: "set_name" + // } + // update_mask { + // paths: "name" + // } + // } + // Please refer to `PutArtifactsRequest` for more details. + optional google.protobuf.FieldMask update_mask = 3; } message PutContextsResponse { @@ -265,12 +507,18 @@ message PutContextsResponse { message PutAttributionsAndAssociationsRequest { repeated Attribution attributions = 1; repeated Association associations = 2; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 3; } message PutAttributionsAndAssociationsResponse {} message PutParentContextsRequest { repeated ParentContext parent_contexts = 1; + + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; } message PutParentContextsResponse {} @@ -283,6 +531,12 @@ message GetArtifactsByTypeRequest { // Currently supports: // 1. Field to order the results. // 2. Page size. + // If set, the request will + // first fetch all artifacts with specified `type_name` and `type_version`, + // then order by a specifield field + // finally find the correct page and return #Artifacts of the page size. + // Higher-level APIs may only use the functionalies partially. + // Please reference the API documentation for the API behaviors. optional ListOperationOptions options = 3; // Options regarding transactions. optional TransactionOptions transaction_options = 4; @@ -312,23 +566,37 @@ message GetArtifactByTypeAndNameResponse { message GetArtifactsByIDRequest { // A list of artifact ids to retrieve. repeated int64 artifact_ids = 1; + // An option to populate all the ArtifactTypes in the response. + // If true, returns retrieved Artifacts and their artifact types, which can be + // matched by type_ids. + // If false, returns only the retrieved Artifacts. + // Example request proto: + // { + // artifact_ids: 101, + // populate_artifact_types: true, + // } + // The response will contain an artifact with id = 101 and an artifact type + // with id = artifact.type_id(). + optional bool populate_artifact_types = 3 [default = false]; // Options regarding transactions. optional TransactionOptions transaction_options = 2; } message GetArtifactsByIDResponse { // Artifacts with matching ids. - // This is not index-aligned: if an id is not found, it is not + // This is not index-aligned: if an id is not found, it is not returned. repeated Artifact artifacts = 1; + // ArtifactTypes populated with matching type_ids owned by `artifacts`. + // This is not index-aligned: if a type_id is not found, it is not returned. + repeated ArtifactType artifact_types = 2; } // Request to retrieve Artifacts using List options. // If option is not specified then all Artifacts are returned. message GetArtifactsRequest { // Specify options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. + // Please refer to the documentation of ListOperationOptions for the supported + // functionalities. optional ListOperationOptions options = 1; // Options regarding transactions. optional TransactionOptions transaction_options = 2; @@ -360,9 +628,8 @@ message GetArtifactsByURIResponse { // If option is not specified then all Executions are returned. message GetExecutionsRequest { // Specify options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. + // Please refer to the documentation of ListOperationOptions for the supported + // functionalities. optional ListOperationOptions options = 1; // Options regarding transactions. optional TransactionOptions transaction_options = 2; @@ -417,6 +684,66 @@ message GetContextTypesResponse { repeated ContextType context_types = 1; } +message GetArtifactsByExternalIdsRequest { + repeated string external_ids = 1; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetArtifactsByExternalIdsResponse { + repeated Artifact artifacts = 1; +} + +message GetExecutionsByExternalIdsRequest { + repeated string external_ids = 1; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetExecutionsByExternalIdsResponse { + repeated Execution executions = 1; +} + +message GetContextsByExternalIdsRequest { + repeated string external_ids = 1; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetContextsByExternalIdsResponse { + repeated Context contexts = 1; +} + +message GetArtifactTypesByExternalIdsRequest { + repeated string external_ids = 1; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetArtifactTypesByExternalIdsResponse { + repeated ArtifactType artifact_types = 1; +} + +message GetExecutionTypesByExternalIdsRequest { + repeated string external_ids = 1; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetExecutionTypesByExternalIdsResponse { + repeated ExecutionType execution_types = 1; +} + +message GetContextTypesByExternalIdsRequest { + repeated string external_ids = 1; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetContextTypesByExternalIdsResponse { + repeated ContextType context_types = 1; +} + message GetExecutionsByTypeRequest { optional string type_name = 1; // If not set, it looks for the type with type_name with default type_version. @@ -425,6 +752,12 @@ message GetExecutionsByTypeRequest { // Currently supports: // 1. Field to order the results. // 2. Page size. + // If set, the request will + // first fetch all executions with specified `type_name` and `type_version`, + // then order by a specifield field + // finally find the correct page and return #Executions of the page size. + // Higher-level APIs may only use the functionalies partially. + // Please reference the API documentation for the API behaviors. optional ListOperationOptions options = 3; // Options regarding transactions. optional TransactionOptions transaction_options = 4; @@ -552,9 +885,8 @@ message GetContextTypesByIDResponse { // If option is not specified then all Contexts are returned. message GetContextsRequest { // Specify options. - // Currently supports: - // 1. Field to order the results. - // 2. Page size. + // Please refer to the documentation of ListOperationOptions for the supported + // functionalities. optional ListOperationOptions options = 1; // Options regarding transactions. optional TransactionOptions transaction_options = 2; @@ -575,6 +907,12 @@ message GetContextsByTypeRequest { // Currently supports: // 1. Field to order the results. // 2. Page size. + // If set, the request will + // first fetch all contexts with specified `type_name` and `type_version`, + // then order by a specifield field + // finally find the correct page and return #Contexts of the page size. + // Higher-level APIs may only use the functionalies partially. + // Please reference the API documentation for the API behaviors. optional ListOperationOptions options = 2; // If not set, it looks for the type with type_name and options with default // type_version. @@ -657,6 +995,32 @@ message GetChildrenContextsByContextResponse { repeated Context contexts = 1; } +message GetParentContextsByContextsRequest { + repeated int64 context_ids = 1 [packed = true]; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetParentContextsByContextsResponse { + message ParentContextsPerChild { + repeated Context parent_contexts = 1; + } + map contexts = 2; +} + +message GetChildrenContextsByContextsRequest { + repeated int64 context_ids = 1 [packed = true]; + // Options regarding transactions. + optional TransactionOptions transaction_options = 2; +} + +message GetChildrenContextsByContextsResponse { + message ChildrenContextsPerParent { + repeated Context children_contexts = 1; + } + map contexts = 2; +} + message GetArtifactsByContextRequest { optional int64 context_id = 1; @@ -700,8 +1064,10 @@ message GetExecutionsByContextResponse { } -// A lineage query request to specify the query nodes of interests and the -// boundary conditions for the returned graph. +// TODO(b/283852485): Deprecate GetLineageGraph API after migration to +// GetLineageSubgraph API. +// A lineage query request to specify the query nodes of interest and the +// boundary conditions for pruning the returned graph. message GetLineageGraphRequest { optional LineageGraphQueryOptions options = 1; // Options regarding transactions. @@ -714,6 +1080,44 @@ message GetLineageGraphResponse { optional LineageGraph subgraph = 1; } +message GetLineageSubgraphRequest { + // Query options for lineage graph tracing from a list of interested + // nodes. + // A lineage subgraph without node details (e.g., external_id, properties) + // will be returned. Please refer to `LineageSubgraphQueryOptions` for more + // details. + optional LineageSubgraphQueryOptions lineage_subgraph_query_options = 1; + // `read_mask` contains user specified paths of fields that should be included + // in the returned lineage subgraph. + // Supported field paths are: 'artifacts', 'executions', 'contexts', + // 'artifact_types', 'execution_types', 'context_types', and 'events'. + // TODO(b/283852485): Include 'associations' or 'attributions' in the + // returned graph. + // If 'artifacts', 'executions', or 'contexts' is specified in `read_mask`, + // the dehydrated nodes will be included. + // Note: A dehydrated node means a node containing only its id and no + // other information. User should call GetNodesByID or other APIs to get + // node details later on. + // If 'artifact_types', 'execution_types', or 'context_types' is specified + // in `read_mask`, all the node types will be included. + // If 'events' is specified in `read_mask`, the events will be included. + // If `read_mask` is not set, the API will return all the fields in + // the returned graph. + // Note: Only paths of fields in LineageGraph message are supported. Paths + // of fields in the submessage, such as "artifacts.id", "contexts.name" are + // not acknowledged. + optional google.protobuf.FieldMask read_mask = 3; + optional TransactionOptions transaction_options = 2; +} + +message GetLineageSubgraphResponse { + // A lineage subgraph of MLMD nodes and relations retrieved from lineage + // graph tracing. + optional LineageGraph lineage_subgraph = 1; +} + + + // LINT.IfChange service MetadataStoreService { // Inserts or updates an ArtifactType. @@ -787,6 +1191,10 @@ service MetadataStoreService { // For new artifacts, type must be specified. // For old artifacts, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated artifacts will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // artifacts: A list of artifacts to insert or update. // @@ -801,6 +1209,10 @@ service MetadataStoreService { // For new executions, type must be specified. // For old executions, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated executions will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // executions: A list of executions to insert or update. // @@ -813,6 +1225,10 @@ service MetadataStoreService { // // The execution_id and artifact_id must already exist. // Once created, events cannot be modified. + // AlreadyExists error will be raised if duplicated events are found. + // + // It is not guaranteed that the created or updated events will share the + // same `milliseconds_since_epoch` timestamps. // // Args: // events: A list of events to insert or update. @@ -824,9 +1240,16 @@ service MetadataStoreService { // input/output Event. The `contexts` describe the associations of the // execution and the attributions of the artifacts. // - // If an execution_id, artifact_id or context_id is specified, it is an - // update, otherwise it does an insertion. For insertion, type must be - // specified. + // If an execution_id is specified, it is an update on the corresponding + // execution, otherwise it does an insertion. + // For insertion, type must be specified. Same rule applies to artifacts + // and contexts in the request. Corresponding errors may raised. For example: + // AlreadyExists error will be raised if duplicated executions, artifacts + // or events are found. + // + // It is not guaranteed that the created or updated executions, artifacts, + // contexts and events will share the same `create_time_since_epoch`, + // `last_update_time_since_epoch`, or `milliseconds_since_epoch` timestamps. // // Args: // execution: An execution to insert or update. @@ -838,6 +1261,38 @@ service MetadataStoreService { // with the input. rpc PutExecution(PutExecutionRequest) returns (PutExecutionResponse) {} + // Inserts or updates a lineage subgraph (i.e. a collection of event edges + // and its executions, artifacts, and related contexts) atomically. The + // `event_edges` include an Event and the indices of the corresponding + // execution and artifact from the input list of executions and artifacts. The + // `contexts` describe the associations of the Execution and the attributions + // of the Artifact. + // + // If an execution_id is specified, it is an update on the corresponding + // Execution, otherwise it does an insertion. For insertion, type must be + // specified. These rules apply to Artifacts and Contexts as well. + // Corresponding errors may be raised. For example: AlreadyExists error will + // be raised if duplicated executions, artifacts, or events are found. + // + // It is not guaranteed that the created or updated executions, artifacts, + // contexts and events will share the same `create_time_since_epoch`, + // `last_update_time_since_epoch`, or `milliseconds_since_epoch` timestamps. + // + // Args: + // executions: A list of executions to insert or update. + // artifacts: A list of artifacts to insert or update. + // contexts: A list of contexts to insert and/or create associations and + // attributions with. + // event_edges: A list of events to insert with the indices of the + // corresponding execution and artifact from the input lists of + // executions and artifacts. + // + // Returns: + // Lists of execution, artifact, and context ids index-aligned with the + // inputs. + rpc PutLineageSubgraph(PutLineageSubgraphRequest) + returns (PutLineageSubgraphResponse) {} + // Inserts or updates contexts in database and returns a list of context ids. // // If an context_id is specified for a context, it is an update. @@ -845,6 +1300,10 @@ service MetadataStoreService { // For new contexts, type must be specified. // For old contexts, type must be unchanged or unspecified. // + // It is not guaranteed that the created or updated contexts will share the + // same `create_time_since_epoch` or `last_update_time_since_epoch` + // timestamps. + // // Args: // contexts: A list of contexts to insert or update. // @@ -986,6 +1445,30 @@ service MetadataStoreService { rpc GetEventsByArtifactIDs(GetEventsByArtifactIDsRequest) returns (GetEventsByArtifactIDsResponse) {} + // Gets all the artifacts with matching external ids. + rpc GetArtifactsByExternalIds(GetArtifactsByExternalIdsRequest) + returns (GetArtifactsByExternalIdsResponse) {} + + // Gets all the artifacts with matching external ids. + rpc GetExecutionsByExternalIds(GetExecutionsByExternalIdsRequest) + returns (GetExecutionsByExternalIdsResponse) {} + + // Gets all the artifacts with matching external ids. + rpc GetContextsByExternalIds(GetContextsByExternalIdsRequest) + returns (GetContextsByExternalIdsResponse) {} + + // Gets all the artifacts with matching external ids. + rpc GetArtifactTypesByExternalIds(GetArtifactTypesByExternalIdsRequest) + returns (GetArtifactTypesByExternalIdsResponse) {} + + // Gets all the artifacts with matching external ids. + rpc GetExecutionTypesByExternalIds(GetExecutionTypesByExternalIdsRequest) + returns (GetExecutionTypesByExternalIdsResponse) {} + + // Gets all the artifacts with matching external ids. + rpc GetContextTypesByExternalIds(GetContextTypesByExternalIdsRequest) + returns (GetContextTypesByExternalIdsResponse) {} + // Gets all context that an artifact is attributed to. rpc GetContextsByArtifact(GetContextsByArtifactRequest) @@ -1003,6 +1486,15 @@ service MetadataStoreService { rpc GetChildrenContextsByContext(GetChildrenContextsByContextRequest) returns (GetChildrenContextsByContextResponse) {} + // Batch getting all the parent contexts that a list of contexts are related. + rpc GetParentContextsByContexts(GetParentContextsByContextsRequest) + returns (GetParentContextsByContextsResponse) {} + + // Batch getting all the children contexts that a list of contexts are + // related. + rpc GetChildrenContextsByContexts(GetChildrenContextsByContextsRequest) + returns (GetChildrenContextsByContextsResponse) {} + // Gets all direct artifacts that a context attributes to. rpc GetArtifactsByContext(GetArtifactsByContextRequest) returns (GetArtifactsByContextResponse) {} @@ -1012,10 +1504,21 @@ service MetadataStoreService { returns (GetExecutionsByContextResponse) {} + // TODO(b/283852485): Deprecate GetLineageGraph API after migration to + // GetLineageSubgraph API. // The transaction performs a constrained transitive closure and returns a // lineage subgraph satisfying the conditions and constraints specified in // the GetLineageGraphRequest. rpc GetLineageGraph(GetLineageGraphRequest) returns (GetLineageGraphResponse) {} + + // Gets a lineage subgraph by performing graph traversal from a list of + // interested nodes. + // A lineage subgraph without node details (e.g., external_id, properties) + // will be returned. + rpc GetLineageSubgraph(GetLineageSubgraphRequest) + returns (GetLineageSubgraphResponse) {} + + } // LINT.ThenChange(../metadata_store/metadata_store_service_interface.h) diff --git a/third_party/ml-metadata/update_version.sh b/third_party/ml-metadata/update_version.sh index 9b907e5759..feffc20751 100755 --- a/third_party/ml-metadata/update_version.sh +++ b/third_party/ml-metadata/update_version.sh @@ -20,7 +20,7 @@ image_files=( "${REPO_ROOT}/.cloudbuild.yaml" \ "${REPO_ROOT}/.release.cloudbuild.yaml" \ "${REPO_ROOT}/manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml" \ "${REPO_ROOT}/test/tag_for_hosted.sh" \ - "${REPO_ROOT}/v2/Makefile" \ + "${REPO_ROOT}/backend/src/v2/Makefile" \ ) for i in "${image_files[@]}" do @@ -29,7 +29,7 @@ do done requirement_files=( "${REPO_ROOT}/backend/metadata_writer/requirements.in" \ - "${REPO_ROOT}/v2/test/requirements.txt" + "${REPO_ROOT}/backend/src/v2/test/requirements.txt" ) for i in "${requirement_files[@]}" do